From 7df1ff35030d52839586dfc9968aa78e11561c87 Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Thu, 23 Sep 2021 04:23:23 +0300
Subject: [PATCH 01/14] add: upload_pictures() prototype

---
 main.py          | 31 +++++++++++++++++++++++++++++--
 requirements.txt |  2 ++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/main.py b/main.py
index 80cccb0..efdd438 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,9 @@
-from fastapi import FastAPI
+from fastapi import FastAPI, File, UploadFile, Depends
 from fastapi.responses import JSONResponse
+from fastapi.security import HTTPBasic, HTTPBasicCredentials
 from fastapi.middleware.cors import CORSMiddleware  # CORS
 
+from secrets import compare_digest
 from datetime import datetime
 from uuid import uuid4
 import sqlite3
@@ -9,10 +11,11 @@ import sqlite3
 
 # use database residing here
 DB_LOCATION = (
-    "db/photovoter.dblite"  # Q: any allowances for this being not OUR database?
+    "../testbox/photovoter.dblite"  # Q: any allowances for this being not OUR database?
 )
 
 app = FastAPI()
+security = HTTPBasic()
 con = sqlite3.connect(DB_LOCATION)
 con.row_factory = sqlite3.Row
 cur = con.cursor()  # NB! single is enough for now, we might require multiple later
@@ -172,3 +175,27 @@ async def photo_points():
         }
         for point in points
     ]
+@app.post(
+    "/upload_pictures/",
+    responses={
+        401: {"description": "Authentication is required to access this resource"},
+        415: {"description": "Cannot process uploaded archive"},
+    },
+)
+async def upload_pictures(
+    credentials: HTTPBasicCredentials = Depends(security), file: UploadFile = File(...)
+):
+    """Интерфейс для загрузки фотографий"""
+    """Условно кладём в браузер zip с фотографиями и он их потихоньку ест. 
+    Доступ к этому интерфейсу, наверное, лучше ограничить паролем или как-нибудь ещё.
+    Пока исходим из предположения, что только я буду загружать фотографии."""
+    # check authenticity
+    correct_username = compare_digest(credentials.username, "1")
+    correct_password = compare_digest(credentials.password, "1")
+    if not (correct_username and correct_password):
+        return JSONResponse(status_code=401)
+    # slurp the zip
+    # *detach from the interface, if possible
+    # unpack zip
+    # feed the pictures to util/import_photos.py
+    return {"filename": file.filename}
diff --git a/requirements.txt b/requirements.txt
index 58ce903..8c2bdf8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,9 @@ pathspec==0.8.1
 pycodestyle==2.7.0
 pydantic==1.8.2
 pyflakes==2.3.1
+python-multipart==0.0.5
 regex==2021.4.4
+six==1.16.0
 starlette==0.14.2
 toml==0.10.2
 typing-extensions==3.10.0.0

From 36cd3297af3cb6f9fc268d6b8ea7216a29fb64ab Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Thu, 23 Sep 2021 21:31:18 +0300
Subject: [PATCH 02/14] add: accept and extract zip  photo archives

---
 main.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 49 insertions(+), 5 deletions(-)

diff --git a/main.py b/main.py
index efdd438..94365ed 100644
--- a/main.py
+++ b/main.py
@@ -1,4 +1,4 @@
-from fastapi import FastAPI, File, UploadFile, Depends
+from fastapi import FastAPI, File, UploadFile, Depends, BackgroundTasks
 from fastapi.responses import JSONResponse
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
 from fastapi.middleware.cors import CORSMiddleware  # CORS
@@ -7,12 +7,14 @@ from secrets import compare_digest
 from datetime import datetime
 from uuid import uuid4
 import sqlite3
+import zipfile
 
 
 # use database residing here
 DB_LOCATION = (
     "../testbox/photovoter.dblite"  # Q: any allowances for this being not OUR database?
 )
+DATA_LOCATION = "/tmp/123"
 
 app = FastAPI()
 security = HTTPBasic()
@@ -178,24 +180,66 @@ async def photo_points():
 @app.post(
     "/upload_pictures/",
     responses={
+        202: {"description": "Archive accepted into processing"},
         401: {"description": "Authentication is required to access this resource"},
         415: {"description": "Cannot process uploaded archive"},
     },
 )
 async def upload_pictures(
-    credentials: HTTPBasicCredentials = Depends(security), file: UploadFile = File(...)
+    background_tasks: BackgroundTasks,
+    credentials: HTTPBasicCredentials = Depends(security),
+    file: UploadFile = File(...),
 ):
     """Интерфейс для загрузки фотографий"""
     """Условно кладём в браузер zip с фотографиями и он их потихоньку ест. 
     Доступ к этому интерфейсу, наверное, лучше ограничить паролем или как-нибудь ещё.
-    Пока исходим из предположения, что только я буду загружать фотографии."""
+    Пока исходим из предположения, что только я буду загружать фотографии.
+    """
     # check authenticity
     correct_username = compare_digest(credentials.username, "1")
     correct_password = compare_digest(credentials.password, "1")
     if not (correct_username and correct_password):
         return JSONResponse(status_code=401)
     # slurp the zip
-    # *detach from the interface, if possible
+    if not zipfile.is_zipfile(file.file):
+        return JSONResponse(status_code=415)
+    # detach from the interface
     # unpack zip
+    tasks = BackgroundTasks()
+    tasks.add_task(
+        unpack_pictures_zip,
+        file=file,
+        time=datetime.utcnow().replace(microsecond=0).isoformat(),
+    )
+
     # feed the pictures to util/import_photos.py
-    return {"filename": file.filename}
+    return JSONResponse("Accepted", background=tasks)
+
+
+def unpack_pictures_zip(file: UploadFile, time):
+    """
+    Unpack and process zip archived photo
+    Extract pictures in the DATA_LOCATION/processing 
+    #TODO: and feed them to util/import_photos.py
+    #TODO: Walk the nested DATA_LOCATION/processing ourselves 
+    Uses: DATA_LOCATION
+    """
+    # we only call this function sporadically, so import here
+    import os
+
+    print(f"Accepted {file.filename} at {time} into processing")
+    os.chdir(DATA_LOCATION)
+    os.mkdir("processing")
+    os.chdir("processing")
+
+    # using private ._file field is a dirty hack, but
+    # SpooledTemporaryFile does not implement seekable
+    # required by zipfile 'r' mode
+    # https://bugs.python.org/issue26175
+    with zipfile.ZipFile(file.file._file) as photo_zip:
+        problem_files = photo_zip.testzip()
+        if problem_files is not None:
+            print(f"Errors in {file.filename} accepted at {time}: {problem_files}")
+        photo_zip.extractall()
+        photo_zip.close()
+    print(f"Succesfully processed {file.filename} accepted at {time}")

From b86efb2154028f4dbcbe531179fbf1f746db996e Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Fri, 24 Sep 2021 01:17:23 +0300
Subject: [PATCH 03/14] ref: pass global params into fn explicitly

---
 util/import_photos.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/util/import_photos.py b/util/import_photos.py
index 906430d..34ca4d3 100644
--- a/util/import_photos.py
+++ b/util/import_photos.py
@@ -23,12 +23,11 @@ def usage():
     print("USAGE: python {name} /path/to/images".format(name=argv[0]), file=stderr)
 
 
-def process_pictures():
+def process_pictures(dest_strunk: str, dest_original: str):
     """Process images from the base directory in the first command line argument.
-    Place the resized copies to DEST_SHRUNK and
-    move the originals to DEST_ORIGINAL.
+    Place the resized copies to DEST_STRUNK and
+    move the originals to dest_original.
     Return a dict for each image processed for database collection.
-    Uses: DEST_SHRUNK, DEST_ORIGINAL
     """
     # walk every pic
     # We only care about files in the root of the path
@@ -50,16 +49,16 @@ def process_pictures():
                 cloned.strip()  # Q: may damage icc, do we allow that or use smh else?
                 cloned.transform(resize="50%")  # Q: what do we want here?
                 # move them to the processed folder
-                cloned.save(filename=path.join(DEST_SHRUNK, filename))
+                cloned.save(filename=path.join(dest_strunk, filename))
 
         # move the originals out of the working directory
         # Q: do we strip exif from originals?
-        move(path.join(root, filename), DEST_ORIGINAL)
+        move(path.join(root, filename), dest_original)
 
         # return the freshly processed picture info
         yield {
-            "ResizedImage": path.join(DEST_SHRUNK, filename),
-            "OriginalImage": path.join(DEST_ORIGINAL, filename),
+            "ResizedImage": path.join(dest_strunk, filename),
+            "OriginalImage": path.join(dest_original, filename),
             "DateTimeOriginal": exif["DateTimeOriginal"],  # Q: normalize it?
             "GPSLatitude": exif["GPSLatitude"],
             "GPSLatitudeRef": exif["GPSLatitudeRef"],
@@ -68,16 +67,15 @@ def process_pictures():
         }
 
 
-def update_database(pic_info: dict):
+def update_database(pic_info: dict, db_location: str):
     """Append new image information to the existing database
     or create a new one, if it does not exist yet
-    Uses: DB_LOCATION
     """
     # make sure the database exists
-    check_database(DB_LOCATION)
+    check_database(db_location)
 
     # FIXME[1]: closure it, so we open it only once?
-    con = sqlite3.connect(DB_LOCATION)
+    con = sqlite3.connect(db_location)
     cur = con.cursor()
     # insert new pictures to the image table
     cur.execute(
@@ -176,8 +174,8 @@ def main():
 
     pics_processed = 0
     # process each pic and add it to the database
-    for pic in process_pictures():
-        update_database(pic)
+    for pic in process_pictures(DEST_STRUNK, DEST_ORIGINAL):
+        update_database(pic, DB_LOCATION)
         pics_processed += 1
 
     if pics_processed == 0:

From ffc3ce75ccfbdd6b5ecb0cab197daed45279ca90 Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Fri, 24 Sep 2021 02:17:58 +0300
Subject: [PATCH 04/14] ref: pass cmdline params into fn explicitly

---
 util/import_photos.py | 41 +++++++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/util/import_photos.py b/util/import_photos.py
index 34ca4d3..e359d77 100644
--- a/util/import_photos.py
+++ b/util/import_photos.py
@@ -18,21 +18,16 @@ DEST_SHRUNK = "db/image/"
 DEST_ORIGINAL = "db/original/"
 
 
-def usage():
-    """Brief usage explanation"""
-    print("USAGE: python {name} /path/to/images".format(name=argv[0]), file=stderr)
-
-
-def process_pictures(dest_strunk: str, dest_original: str):
+def process_pictures(source: str, dest_shrunk: str, dest_original: str):
     """Process images from the base directory in the first command line argument.
-    Place the resized copies to DEST_STRUNK and
+    Place the resized copies to dest_shrunk and
     move the originals to dest_original.
     Return a dict for each image processed for database collection.
     """
     # walk every pic
     # We only care about files in the root of the path
     # Ignore any nested directories
-    (root, _, filenames) = next(walk(argv[1], topdown=True), (None, None, []))
+    (root, _, filenames) = next(walk(source, topdown=True), (None, None, []))
     for filename in filenames:
         # FIXME[0]:what if picture with the same name already exists?
         # skip any non-image files
@@ -49,7 +44,7 @@ def process_pictures(dest_strunk: str, dest_original: str):
                 cloned.strip()  # Q: may damage icc, do we allow that or use smh else?
                 cloned.transform(resize="50%")  # Q: what do we want here?
                 # move them to the processed folder
-                cloned.save(filename=path.join(dest_strunk, filename))
+                cloned.save(filename=path.join(dest_shrunk, filename))
 
         # move the originals out of the working directory
         # Q: do we strip exif from originals?
@@ -57,7 +52,7 @@ def process_pictures(dest_strunk: str, dest_original: str):
 
         # return the freshly processed picture info
         yield {
-            "ResizedImage": path.join(dest_strunk, filename),
+            "ResizedImage": path.join(dest_shrunk, filename),
             "OriginalImage": path.join(dest_original, filename),
             "DateTimeOriginal": exif["DateTimeOriginal"],  # Q: normalize it?
             "GPSLatitude": exif["GPSLatitude"],
@@ -167,23 +162,33 @@ def check_database(database_path: str):
         con.close()
 
 
-def main():
-    if len(argv) != 2:
-        usage()
-        exit(1)
-
+def run(db_location: str, source: str, dest_shrunk: str, dest_original: str):
+    """Core program logic"""
     pics_processed = 0
     # process each pic and add it to the database
-    for pic in process_pictures(DEST_STRUNK, DEST_ORIGINAL):
-        update_database(pic, DB_LOCATION)
+    for pic in process_pictures(source, dest_shrunk, dest_original):
+        update_database(pic, db_location)
         pics_processed += 1
 
     if pics_processed == 0:
-        print("No more pictures processed from", argv[1])
+        print("No more pictures processed from", source)
         print("Do we have enough permissions?")
     else:
         print("Pictures processed:", pics_processed)
 
 
+def usage():
+    """Brief usage explanation"""
+    print("USAGE: python {name} /path/to/images".format(name=argv[0]), file=stderr)
+
+
+def main():
+    if len(argv) != 2:
+        usage()
+        exit(1)
+
+    run(DB_LOCATION, argv[1], DEST_SHRUNK, DEST_ORIGINAL)
+
+
 if __name__ == "__main__":
     main()

From 822e512a01d3433b935d09c3a9ae721e7c6d3ed2 Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Fri, 24 Sep 2021 02:47:48 +0300
Subject: [PATCH 05/14] ref: extract globals into common config.py

---
 config.py             | 11 +++++++++++
 main.py               | 15 ++++++---------
 util/import_photos.py | 19 +++++++++----------
 3 files changed, 26 insertions(+), 19 deletions(-)
 create mode 100644 config.py

diff --git a/config.py b/config.py
new file mode 100644
index 0000000..29e6f07
--- /dev/null
+++ b/config.py
@@ -0,0 +1,11 @@
+# use database residing here
+DB_LOCATION = (
+    "testbox/photovoter.dblite"  # Q: any allowances for this being not OUR database?
+)
+
+DATA_LOCATION = "/tmp/123"
+
+# place compressed images here (needs to exist)
+DEST_SHRUNK = "image/"
+# move originals here (needs to exist)
+DEST_ORIGINAL = "original/"
diff --git a/main.py b/main.py
index 94365ed..ccd852f 100644
--- a/main.py
+++ b/main.py
@@ -9,12 +9,9 @@ from uuid import uuid4
 import sqlite3
 import zipfile
 
-
-# use database residing here
-DB_LOCATION = (
-    "../testbox/photovoter.dblite"  # Q: any allowances for this being not OUR database?
-)
-DATA_LOCATION = "/tmp/123"
+# Global settings of this program
+# ./config.py
+from config import DB_LOCATION, DATA_LOCATION
 
 app = FastAPI()
 security = HTTPBasic()
@@ -219,9 +216,9 @@ async def upload_pictures(
 def unpack_pictures_zip(file: UploadFile, time):
     """
     Unpack and process zip archived photo
-    Extract pictures in the DATA_LOCATION/processing 
+    Extract pictures in the DATA_LOCATION/processing
     #TODO: and feed them to util/import_photos.py
-    #TODO: Walk the nested DATA_LOCATION/processing ourselves 
+    #TODO: Walk the nested DATA_LOCATION/processing ourselves
     Uses: DATA_LOCATION
     """
     # we only call this function sporadically, so import here
@@ -229,7 +226,7 @@ def unpack_pictures_zip(file: UploadFile, time):
 
     print(f"Accepted {file.filename} at {time} into processing")
     os.chdir(DATA_LOCATION)
-    os.mkdir("processing")
+    os.makedirs("processing", exist_ok=True)
     os.chdir("processing")
 
     # using private ._file field is a dirty hack, but
diff --git a/util/import_photos.py b/util/import_photos.py
index e359d77..b165a1a 100644
--- a/util/import_photos.py
+++ b/util/import_photos.py
@@ -8,15 +8,6 @@ from sys import argv, stderr
 from shutil import move
 import sqlite3
 
-# update database residing here
-DB_LOCATION = (
-    "db/photovoter.dblite"  # Q: any allowances for this being not OUR database?
-)
-# place compressed images here (needs to exist)
-DEST_SHRUNK = "db/image/"
-# move originals here (needs to exist)
-DEST_ORIGINAL = "db/original/"
-
 
 def process_pictures(source: str, dest_shrunk: str, dest_original: str):
     """Process images from the base directory in the first command line argument.
@@ -187,7 +178,15 @@ def main():
         usage()
         exit(1)
 
-    run(DB_LOCATION, argv[1], DEST_SHRUNK, DEST_ORIGINAL)
+    import sys
+    import os
+
+    # append root directory to sys.path
+    # to allow import globals from ../config.py
+    sys.path.append(os.path.dirname(__file__) + "/..")
+    import config as cfg
+
+    run(cfg.DB_LOCATION, argv[1], cfg.DEST_SHRUNK, cfg.DEST_ORIGINAL)
 
 
 if __name__ == "__main__":

From 250f09153d2500e358a950a6e122e0f1c950f31e Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Sun, 26 Sep 2021 22:32:52 +0300
Subject: [PATCH 06/14] ref: use explicit paths instead of chdir

---
 main.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/main.py b/main.py
index ccd852f..90da7c2 100644
--- a/main.py
+++ b/main.py
@@ -225,9 +225,7 @@ def unpack_pictures_zip(file: UploadFile, time):
     import os
 
     print(f"Accepted {file.filename} at {time} into processing")
-    os.chdir(DATA_LOCATION)
-    os.makedirs("processing", exist_ok=True)
-    os.chdir("processing")
+    os.makedirs(os.path.join(DATA_LOCATION, "processing"), exist_ok=True)
 
     # using private ._file field is a dirty hack, but
     # SpooledTemporaryFile does not implement seekable
@@ -236,7 +234,10 @@ def unpack_pictures_zip(file: UploadFile, time):
     with zipfile.ZipFile(file.file._file) as photo_zip:
         problem_files = photo_zip.testzip()
         if problem_files is not None:
-            print(f"Errors in {file.filename} accepted at {time}: {problem_files}")
-        photo_zip.extractall()
+            print(f"Errors in {file.filename} from {time} at {problem_files}")
+        photo_zip.extractall(path=os.path.join(DATA_LOCATION, "processing"))
         photo_zip.close()
-    print(f"Succesfully processed {file.filename} accepted at {time}")
+
+    print(f"Start processing {file.filename} from {time}")
+
+    print(f"Succesfully processed {file.filename} from {time}")

From 54a7010e7613c630b90cc3d82a47d7acd18bbf3a Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Mon, 27 Sep 2021 04:52:37 +0300
Subject: [PATCH 07/14] add: process uploaded photo

---
 main.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/main.py b/main.py
index 90da7c2..6815ffd 100644
--- a/main.py
+++ b/main.py
@@ -11,10 +11,16 @@ import zipfile
 
 # Global settings of this program
 # ./config.py
-from config import DB_LOCATION, DATA_LOCATION
+from config import DB_LOCATION, DATA_LOCATION, DEST_SHRUNK, DEST_ORIGINAL
 
+
+# our own util for photo upload and processing
+from util import import_photos as iph
+
+# Initialization logic
 app = FastAPI()
 security = HTTPBasic()
+iph.check_database(database_path=DB_LOCATION)
 con = sqlite3.connect(DB_LOCATION)
 con.row_factory = sqlite3.Row
 cur = con.cursor()  # NB! single is enough for now, we might require multiple later
@@ -219,13 +225,15 @@ def unpack_pictures_zip(file: UploadFile, time):
     Extract pictures in the DATA_LOCATION/processing
     #TODO: and feed them to util/import_photos.py
     #TODO: Walk the nested DATA_LOCATION/processing ourselves
-    Uses: DATA_LOCATION
+    Uses: DB_LOCATION, DATA_LOCATION
     """
     # we only call this function sporadically, so import here
     import os
+    from shutil import rmtree
 
     print(f"Accepted {file.filename} at {time} into processing")
-    os.makedirs(os.path.join(DATA_LOCATION, "processing"), exist_ok=True)
+    processing_path = os.path.join(DATA_LOCATION, "processing" + time)
+    os.makedirs(processing_path, exist_ok=True)
 
     # using private ._file field is a dirty hack, but
     # SpooledTemporaryFile does not implement seekable
@@ -235,9 +243,20 @@ def unpack_pictures_zip(file: UploadFile, time):
         problem_files = photo_zip.testzip()
         if problem_files is not None:
             print(f"Errors in {file.filename} from {time} at {problem_files}")
-        photo_zip.extractall(path=os.path.join(DATA_LOCATION, "processing"))
+        photo_zip.extractall(path=processing_path)
         photo_zip.close()
 
     print(f"Start processing {file.filename} from {time}")
 
+    iph.check_database(database_path=DB_LOCATION)
+    for (dir, _, _) in os.walk(processing_path):
+        iph.run(
+            db_location=DB_LOCATION,
+            source=os.path.join(dir),
+            dest_shrunk=os.path.join(DATA_LOCATION, DEST_SHRUNK),
+            dest_original=os.path.join(DATA_LOCATION, DEST_ORIGINAL),
+        )
+
+    rmtree(processing_path)
+
     print(f"Succesfully processed {file.filename} from {time}")

From 4b0b6ab9cc6a07b6ca45993ba7755b0bd623b397 Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Mon, 27 Sep 2021 05:01:00 +0300
Subject: [PATCH 08/14] ref: extract upload credentials into config

---
 config.py |  4 ++++
 main.py   | 13 ++++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/config.py b/config.py
index 29e6f07..62eab32 100644
--- a/config.py
+++ b/config.py
@@ -9,3 +9,7 @@ DATA_LOCATION = "/tmp/123"
 DEST_SHRUNK = "image/"
 # move originals here (needs to exist)
 DEST_ORIGINAL = "original/"
+
+# upload interface credentials
+CRED_USERNAME="changeme"
+CRED_PASSWORD="CHANGEME"
\ No newline at end of file
diff --git a/main.py b/main.py
index 6815ffd..a6f68cc 100644
--- a/main.py
+++ b/main.py
@@ -11,7 +11,14 @@ import zipfile
 
 # Global settings of this program
 # ./config.py
-from config import DB_LOCATION, DATA_LOCATION, DEST_SHRUNK, DEST_ORIGINAL
+from config import (
+    DB_LOCATION,
+    DATA_LOCATION,
+    DEST_SHRUNK,
+    DEST_ORIGINAL,
+    CRED_USERNAME,
+    CRED_PASSWORD,
+)
 
 
 # our own util for photo upload and processing
@@ -199,8 +206,8 @@ async def upload_pictures(
     Пока исходим из предположения, что только я буду загружать фотографии.
     """
     # check authenticity
-    correct_username = compare_digest(credentials.username, "1")
-    correct_password = compare_digest(credentials.password, "1")
+    correct_username = compare_digest(credentials.username, CRED_USERNAME)
+    correct_password = compare_digest(credentials.password, CRED_PASSWORD)
     if not (correct_username and correct_password):
         return JSONResponse(status_code=401)
     # slurp the zip

From 3e35188739bc016f439dcdca0a164aa158873f9b Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Mon, 27 Sep 2021 05:33:16 +0300
Subject: [PATCH 09/14] msc: upload_pictures() docs

---
 config.py |  4 ++--
 main.py   | 16 +++++++++-------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/config.py b/config.py
index 62eab32..5e5109b 100644
--- a/config.py
+++ b/config.py
@@ -11,5 +11,5 @@ DEST_SHRUNK = "image/"
 DEST_ORIGINAL = "original/"
 
 # upload interface credentials
-CRED_USERNAME="changeme"
-CRED_PASSWORD="CHANGEME"
\ No newline at end of file
+CRED_USERNAME = "changeme"
+CRED_PASSWORD = "CHANGEME"
diff --git a/main.py b/main.py
index a6f68cc..8cf6443 100644
--- a/main.py
+++ b/main.py
@@ -188,7 +188,7 @@ async def photo_points():
         for point in points
     ]
 @app.post(
-    "/upload_pictures/",
+    "/upload_pictures",
     responses={
         202: {"description": "Archive accepted into processing"},
         401: {"description": "Authentication is required to access this resource"},
@@ -200,10 +200,12 @@ async def upload_pictures(
     credentials: HTTPBasicCredentials = Depends(security),
     file: UploadFile = File(...),
 ):
-    """Интерфейс для загрузки фотографий"""
-    """Условно кладём в браузер zip с фотографиями и он их потихоньку ест. 
-    Доступ к этому интерфейсу, наверное, лучше ограничить паролем или как-нибудь ещё.
-    Пока исходим из предположения, что только я буду загружать фотографии.
+    """Photo upload endpoint"""
+    """
+    Accepts photo in zip archives with any internal directory structure
+    Valid uploads yield 202 status message and process photos in the background
+    Non-zip archives yeild 415 error
+    Upload is restricted by basic HTTP login, configurable in config.py
     """
     # check authenticity
     correct_username = compare_digest(credentials.username, CRED_USERNAME)
@@ -230,8 +232,8 @@ def unpack_pictures_zip(file: UploadFile, time):
     """
     Unpack and process zip archived photo
     Extract pictures in the DATA_LOCATION/processing
-    #TODO: and feed them to util/import_photos.py
-    #TODO: Walk the nested DATA_LOCATION/processing ourselves
+    and feed them to util/import_photos.py
+    Walk the nested DATA_LOCATION/processing ourselves
     Uses: DB_LOCATION, DATA_LOCATION
     """
     # we only call this function sporadically, so import here

From 576b23b2c9b5b239b6b0dd56f65e0c2b85bb3e57 Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Mon, 27 Sep 2021 05:34:05 +0300
Subject: [PATCH 10/14] fix: add 202 status code to upload_pictures()

---
 main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/main.py b/main.py
index 8cf6443..eebb09b 100644
--- a/main.py
+++ b/main.py
@@ -225,7 +225,7 @@ async def upload_pictures(
     )
 
     # feed the pictures to util/import_photos.py
-    return JSONResponse("Accepted", background=tasks)
+    return JSONResponse("Accepted", status_code=202, background=tasks)
 
 
 def unpack_pictures_zip(file: UploadFile, time):

From a1cd63eca072e5b747789b496b9542c55efd1b0b Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Sun, 19 Dec 2021 02:51:45 +0300
Subject: [PATCH 11/14] fix: switch to epoch seconds to allow for win path

---
 main.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/main.py b/main.py
index eebb09b..3708655 100644
--- a/main.py
+++ b/main.py
@@ -187,6 +187,8 @@ async def photo_points():
         }
         for point in points
     ]
+
+
 @app.post(
     "/upload_pictures",
     responses={
@@ -221,7 +223,7 @@ async def upload_pictures(
     tasks.add_task(
         unpack_pictures_zip,
         file=file,
-        time=datetime.utcnow().replace(microsecond=0).isoformat(),
+        time=datetime.utcnow().replace(microsecond=0),
     )
 
     # feed the pictures to util/import_photos.py
@@ -240,8 +242,8 @@ def unpack_pictures_zip(file: UploadFile, time):
     import os
     from shutil import rmtree
 
-    print(f"Accepted {file.filename} at {time} into processing")
-    processing_path = os.path.join(DATA_LOCATION, "processing" + time)
+    print(f"Accepted {file.filename} at {time.isoformat()} into processing")
+    processing_path = os.path.join(DATA_LOCATION, "processing" + str(time.timestamp()))
     os.makedirs(processing_path, exist_ok=True)
 
     # using private ._file field is a dirty hack, but
@@ -251,11 +253,13 @@ def unpack_pictures_zip(file: UploadFile, time):
     with zipfile.ZipFile(file.file._file) as photo_zip:
         problem_files = photo_zip.testzip()
         if problem_files is not None:
-            print(f"Errors in {file.filename} from {time} at {problem_files}")
+            print(
+                f"Errors in {file.filename} from {time.isoformat()} at {problem_files}"
+            )
         photo_zip.extractall(path=processing_path)
         photo_zip.close()
 
-    print(f"Start processing {file.filename} from {time}")
+    print(f"Start processing {file.filename} from {time.isoformat()}")
 
     iph.check_database(database_path=DB_LOCATION)
     for (dir, _, _) in os.walk(processing_path):
@@ -268,4 +272,4 @@ def unpack_pictures_zip(file: UploadFile, time):
 
     rmtree(processing_path)
 
-    print(f"Succesfully processed {file.filename} from {time}")
+    print(f"Succesfully processed {file.filename} from {time.isoformat()}")

From b78c53441dc62fd2db50d288e38f59731aa04cab Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Sun, 19 Dec 2021 04:13:36 +0300
Subject: [PATCH 12/14] add: skip adding images with invalid exif to db

---
 util/import_photos.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/util/import_photos.py b/util/import_photos.py
index b165a1a..a14e7f0 100644
--- a/util/import_photos.py
+++ b/util/import_photos.py
@@ -41,16 +41,20 @@ def process_pictures(source: str, dest_shrunk: str, dest_original: str):
         # Q: do we strip exif from originals?
         move(path.join(root, filename), dest_original)
 
-        # return the freshly processed picture info
-        yield {
-            "ResizedImage": path.join(dest_shrunk, filename),
-            "OriginalImage": path.join(dest_original, filename),
-            "DateTimeOriginal": exif["DateTimeOriginal"],  # Q: normalize it?
-            "GPSLatitude": exif["GPSLatitude"],
-            "GPSLatitudeRef": exif["GPSLatitudeRef"],
-            "GPSLongitude": exif["GPSLongitude"],
-            "GPSLongitudeRef": exif["GPSLongitudeRef"],
-        }
+        try:
+            # return the freshly processed picture info
+            yield {
+                "ResizedImage": path.join(dest_shrunk, filename),
+                "OriginalImage": path.join(dest_original, filename),
+                "DateTimeOriginal": exif["DateTimeOriginal"],  # Q: normalize it?
+                "GPSLatitude": exif["GPSLatitude"],
+                "GPSLatitudeRef": exif["GPSLatitudeRef"],
+                "GPSLongitude": exif["GPSLongitude"],
+                "GPSLongitudeRef": exif["GPSLongitudeRef"],
+            }
+        except KeyError as e:
+            print(f"Image '{filename}' has no valid exif")
+            continue
 
 
 def update_database(pic_info: dict, db_location: str):

From 3e7ef81102b53076c0d920ea96e81bd637eafea6 Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Sun, 19 Dec 2021 04:15:15 +0300
Subject: [PATCH 13/14] msc: report abs db path on creation

---
 util/import_photos.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/util/import_photos.py b/util/import_photos.py
index a14e7f0..e65fbca 100644
--- a/util/import_photos.py
+++ b/util/import_photos.py
@@ -102,7 +102,7 @@ def check_database(database_path: str):
         return
     # make one
     else:
-        print("No DB, creating", database_path)
+        print("No DB, creating", path.abspath(database_path))
 
         con = sqlite3.connect(database_path)
         cur = con.cursor()
@@ -166,7 +166,7 @@ def run(db_location: str, source: str, dest_shrunk: str, dest_original: str):
         pics_processed += 1
 
     if pics_processed == 0:
-        print("No more pictures processed from", source)
+        print("No pictures processed from", source)
         print("Do we have enough permissions?")
     else:
         print("Pictures processed:", pics_processed)

From 9620458977ae03d517e809805f9ed10cbdbfca23 Mon Sep 17 00:00:00 2001
From: rrr-marble <rrr@marble.lan>
Date: Sun, 19 Dec 2021 18:20:07 +0300
Subject: [PATCH 14/14] fix: normalize global rel paths for win compat

---
 main.py               | 4 ++--
 util/import_photos.py | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/main.py b/main.py
index 3708655..d6f9221 100644
--- a/main.py
+++ b/main.py
@@ -266,8 +266,8 @@ def unpack_pictures_zip(file: UploadFile, time):
         iph.run(
             db_location=DB_LOCATION,
             source=os.path.join(dir),
-            dest_shrunk=os.path.join(DATA_LOCATION, DEST_SHRUNK),
-            dest_original=os.path.join(DATA_LOCATION, DEST_ORIGINAL),
+            dest_shrunk=os.path.join(DATA_LOCATION, os.path.normcase(DEST_SHRUNK)),
+            dest_original=os.path.join(DATA_LOCATION, os.path.normcase(DEST_ORIGINAL)),
         )
 
     rmtree(processing_path)
diff --git a/util/import_photos.py b/util/import_photos.py
index e65fbca..4e80210 100644
--- a/util/import_photos.py
+++ b/util/import_photos.py
@@ -190,7 +190,12 @@ def main():
     sys.path.append(os.path.dirname(__file__) + "/..")
     import config as cfg
 
-    run(cfg.DB_LOCATION, argv[1], cfg.DEST_SHRUNK, cfg.DEST_ORIGINAL)
+    run(
+        cfg.DB_LOCATION,
+        argv[1],
+        path.normcase(cfg.DEST_SHRUNK),
+        path.normcase(cfg.DEST_ORIGINAL),
+    )
 
 
 if __name__ == "__main__":