Implemented everything upload

src/utils/__init__.py

@@ -1,3 +1,7 @@
 from .healthchecks import health_database_status
 from .security import security, init_security_real_good
 from .config import Config
+from .storage import storage
+from .md5stuffs import calculate_md5_sum_for_file, write_file_from_stream_to_file_like_while_calculating_md5
+from .exceptions import FileIntegrityError
+from .caff_previewer import create_caff_preview

src/utils/caff_previewer.py

@@ -0,0 +1,56 @@
+import tempfile
+import os
+import os.path
+import shutil
+import requests
+from flask import current_app
+from .md5stuffs import calculate_md5_sum_for_file, write_file_from_stream_to_file_like_while_calculating_md5
+from .exceptions import FileIntegrityError
+import magic
+
+EXPECTED_PREVIEW_MIMETYPE = 'image/png'
+
+# The response is validated for
+# - integrity (2-way)
+# - mime type (both header and file identifying)
+# - size (handled by md5sum saver, generator thingy)
+
+def create_caff_preview(src_file: str) -> str:  # hopefully returns a file path
+
+    # Send file for previewing
+    uploaded_caff_md5sum = calculate_md5_sum_for_file(src_file)
+
+    with open(src_file, 'rb') as f:
+        r = requests.post(current_app.config['CAFF_PREVIEWER_ENDPOINT'], data=f, stream=True)
+
+    r.raise_for_status()
+
+    # Verify the results while saving the file
+    if r.headers.get("Content-type") != EXPECTED_PREVIEW_MIMETYPE:
+        raise ValueError(f"Converter output (reported by header) is not {EXPECTED_PREVIEW_MIMETYPE}")
+
+    if r.headers.get("X-request-checksum") != uploaded_caff_md5sum:
+        # This really is the most pointless check in the world
+        # But it was fun to implement
+        raise FileIntegrityError("File sent for previewing and received by previewer differ")
+
+    converted_png_fd, converted_png_path = tempfile.mkstemp(
+        prefix=os.path.basename(src_file).split('.')[0],
+        suffix='.png'
+    )
+
+    with open(converted_png_fd, "wb") as f:
+        converted_png_md5sum = write_file_from_stream_to_file_like_while_calculating_md5(r.raw, f)
+
+    if r.headers.get("X-response-checksum") != converted_png_md5sum:
+        # This does not have much point either
+        raise FileIntegrityError("File sent by previewer and received by the app differ")
+
+    with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:
+        calculated_mimetype = m.id_filename(converted_png_path)
+
+    if calculated_mimetype != EXPECTED_PREVIEW_MIMETYPE:
+        raise ValueError(f"Converter output (calculated from file) is not {EXPECTED_PREVIEW_MIMETYPE}")
+
+    del r
+    return converted_png_path

src/utils/config.py

@@ -30,7 +30,17 @@ class Config:
     MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
     SECURITY_EMAIL_SENDER = os.environ.get('SECURITY_EMAIL_SENDER', 'noreply@unstablevortex.kmlabz.com')
 
+    CAFF_PREVIEWER_ENDPOINT = os.environ["CAFF_PREVIEWER_ENDPOINT"]  # This should prevent application startup
+
+    # Those all should fail the application startup
+    MINIO_ENDPOINT = os.environ["MINIO_ENDPOINT"]
+    MINIO_ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"]
+    MINIO_SECRET_KEY = os.environ["MINIO_SECRET_KEY"]
+    MINIO_SECURE = os.environ.get("MINIO_SECURE", "true").upper() == 'TRUE'
+
     # Some constant configured stuff configs
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     SECURITY_REGISTERABLE = True
     SECURITY_PASSWORD_HASH = "bcrypt"
+    MINIO_PREVIEW_BUCKET_NAME = "previews"
+    MINIO_CAFF_BUCKET_NAME = "caff"

src/utils/exceptions.py

@@ -0,0 +1,4 @@
+
+
+class FileIntegrityError(BaseException):
+    pass

src/utils/md5stuffs.py

@@ -0,0 +1,34 @@
+import hashlib
+
+
+def write_file_from_stream_to_file_like_while_calculating_md5(stream, f, maxsize: int = 536870912,
+                                                              chunksize: int = 4096) -> str:
+    m = hashlib.md5()  # nosec: md5 is used only for integrity checking here
+
+    total_recieved = 0
+
+    # Begin receiving the file
+
+    while True:  # This is where uploading happens
+        chunk = stream.read(chunksize)
+        if len(chunk) == 0:
+            break
+
+        total_recieved += len(chunk)
+        if total_recieved > maxsize:
+            raise OverflowError("File too big")
+
+        m.update(chunk)
+        f.write(chunk)
+
+    return m.hexdigest()
+
+
+def calculate_md5_sum_for_file(fname) -> str:
+    m = hashlib.md5()  # nosec: md5 is used only for integrity checking here
+
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            m.update(chunk)
+
+    return m.hexdigest()

src/utils/storage.py

@@ -0,0 +1,3 @@
+from flask_minio import Minio
+
+storage = Minio()