Log in to upload an animation.
-{% endif %} + {% if current_user.is_authenticated %} +Log in to upload an animation.
+ {% endif %} {% endblock %} \ No newline at end of file diff --git a/src/utils/__init__.py b/src/utils/__init__.py index 769ee5c..de5a089 100644 --- a/src/utils/__init__.py +++ b/src/utils/__init__.py @@ -1,3 +1,7 @@ from .healthchecks import health_database_status from .security import security, init_security_real_good from .config import Config +from .storage import storage +from .md5stuffs import calculate_md5_sum_for_file, write_file_from_stream_to_file_like_while_calculating_md5 +from .exceptions import FileIntegrityError +from .caff_previewer import create_caff_preview \ No newline at end of file diff --git a/src/utils/caff_previewer.py b/src/utils/caff_previewer.py new file mode 100644 index 0000000..aeb13ae --- /dev/null +++ b/src/utils/caff_previewer.py @@ -0,0 +1,56 @@ +import tempfile +import os +import os.path +import shutil +import requests +from flask import current_app +from .md5stuffs import calculate_md5_sum_for_file, write_file_from_stream_to_file_like_while_calculating_md5 +from .exceptions import FileIntegrityError +import magic + +EXPECTED_PREVIEW_MIMETYPE = 'image/png' + +# The response is validated for +# - integrity (2-way) +# - mime type (both header and file identifying) +# - size (handled by md5sum saver, generator thingy) + +def create_caff_preview(src_file: str) -> str: # hopefully returns a file path + + # Send file for previewing + uploaded_caff_md5sum = calculate_md5_sum_for_file(src_file) + + with open(src_file, 'rb') as f: + r = requests.post(current_app.config['CAFF_PREVIEWER_ENDPOINT'], data=f, stream=True) + + r.raise_for_status() + + # Verify the results while saving the file + if r.headers.get("Content-type") != EXPECTED_PREVIEW_MIMETYPE: + raise ValueError(f"Converter output (reported by header) is not {EXPECTED_PREVIEW_MIMETYPE}") + + if r.headers.get("X-request-checksum") != uploaded_caff_md5sum: + # This really is the most pointless check in the world + # But it was fun to implement + raise FileIntegrityError("File sent for previewing and received by previewer differ") + + converted_png_fd, converted_png_path = tempfile.mkstemp( + prefix=os.path.basename(src_file).split('.')[0], + suffix='.png' + ) + + with open(converted_png_fd, "wb") as f: + converted_png_md5sum = write_file_from_stream_to_file_like_while_calculating_md5(r.raw, f) + + if r.headers.get("X-response-checksum") != converted_png_md5sum: + # This does not have much point either + raise FileIntegrityError("File sent by previewer and received by the app differ") + + with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m: + calculated_mimetype = m.id_filename(converted_png_path) + + if calculated_mimetype != EXPECTED_PREVIEW_MIMETYPE: + raise ValueError(f"Converter output (calculated from file) is not {EXPECTED_PREVIEW_MIMETYPE}") + + del r + return converted_png_path diff --git a/src/utils/config.py b/src/utils/config.py index 0c84124..17d5e18 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -30,7 +30,17 @@ class Config: MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD') SECURITY_EMAIL_SENDER = os.environ.get('SECURITY_EMAIL_SENDER', 'noreply@unstablevortex.kmlabz.com') + CAFF_PREVIEWER_ENDPOINT = os.environ["CAFF_PREVIEWER_ENDPOINT"] # This should prevent application startup + + # Those all should fail the application startup + MINIO_ENDPOINT = os.environ["MINIO_ENDPOINT"] + MINIO_ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"] + MINIO_SECRET_KEY = os.environ["MINIO_SECRET_KEY"] + MINIO_SECURE = os.environ.get("MINIO_SECURE", "true").upper() == 'TRUE' + # Some constant configured stuff configs SQLALCHEMY_TRACK_MODIFICATIONS = False SECURITY_REGISTERABLE = True SECURITY_PASSWORD_HASH = "bcrypt" + MINIO_PREVIEW_BUCKET_NAME = "previews" + MINIO_CAFF_BUCKET_NAME = "caff" diff --git a/src/utils/exceptions.py b/src/utils/exceptions.py new file mode 100644 index 0000000..ed8fe16 --- /dev/null +++ b/src/utils/exceptions.py @@ -0,0 +1,4 @@ + + +class FileIntegrityError(BaseException): + pass \ No newline at end of file diff --git a/src/utils/md5stuffs.py b/src/utils/md5stuffs.py new file mode 100644 index 0000000..15730d1 --- /dev/null +++ b/src/utils/md5stuffs.py @@ -0,0 +1,34 @@ +import hashlib + + +def write_file_from_stream_to_file_like_while_calculating_md5(stream, f, maxsize: int = 536870912, + chunksize: int = 4096) -> str: + m = hashlib.md5() # nosec: md5 is used only for integrity checking here + + total_recieved = 0 + + # Begin receiving the file + + while True: # This is where uploading happens + chunk = stream.read(chunksize) + if len(chunk) == 0: + break + + total_recieved += len(chunk) + if total_recieved > maxsize: + raise OverflowError("File too big") + + m.update(chunk) + f.write(chunk) + + return m.hexdigest() + + +def calculate_md5_sum_for_file(fname) -> str: + m = hashlib.md5() # nosec: md5 is used only for integrity checking here + + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + m.update(chunk) + + return m.hexdigest() diff --git a/src/utils/storage.py b/src/utils/storage.py new file mode 100644 index 0000000..0259793 --- /dev/null +++ b/src/utils/storage.py @@ -0,0 +1,3 @@ +from flask_minio import Minio + +storage = Minio() \ No newline at end of file diff --git a/src/views/uploadview.py b/src/views/uploadview.py index 9c22ab8..3763e13 100644 --- a/src/views/uploadview.py +++ b/src/views/uploadview.py @@ -1,6 +1,17 @@ #!/usr/bin/env python3 -from flask import render_template +import tempfile +import os +import os.path +import minio +from flask import render_template, flash, redirect, url_for, request, current_app from flask_classful import FlaskView +from flask_security import current_user, login_required +from utils import storage +import bleach +from models import db, Item + +from utils import create_caff_preview +from requests import HTTPError """ Upload VIEW @@ -16,3 +27,82 @@ class UploadView(FlaskView): def index(self): return render_template('upload.html') + + @login_required + def post(self): + title = request.form.get('title') + title = title[:Item.name.property.columns[0].type.length] + title = bleach.clean(title, tags=[]) + + if not title: + flash("Title must be filled", "primary") + return redirect(url_for('UploadView:index')) + + if 'file' not in request.files: + flash("No file provided", "primary") + return redirect(url_for('UploadView:index')) + + file = request.files['file'] + uploaded_caff_fd, uploaded_caff_path = tempfile.mkstemp(prefix=current_user.name, suffix='.caff') + + with open(uploaded_caff_fd, "wb") as f: + file.save(f) + + # let the memes begin! ----------------------------------------------------------------------------------------- + success = False + try: + converted_png_path = create_caff_preview(uploaded_caff_path) + success = os.path.isfile(converted_png_path) # check if the file really is there + except HTTPError as e: + if e.response.status_code == 400: + flash("Invalid CAFF file", "danger") + elif e.response.status_code == 413: + flash("CAFF file too large", "danger") + else: + raise # Whatever... we'll just check the Sentry alert + + except OverflowError: + flash("CAFF file too large", "danger") + + except ValueError: + flash("Something went wrong, try again later...", "warning") + + if not success: + os.unlink(uploaded_caff_path) + return redirect(url_for('UploadView:index')) + + # End of the meme part ----------------------------------------------------------------------------------------- + + # Upload everything to minio + item = Item(name=title, uploader=current_user) + db.session.add(item) + db.session.flush() # To obtain an id + + try: + storage.connection.make_bucket(current_app.config['MINIO_PREVIEW_BUCKET_NAME']) + except minio.error.BucketAlreadyOwnedByYou: + pass + + storage.connection.fput_object( + current_app.config['MINIO_PREVIEW_BUCKET_NAME'], + str(item.id), + converted_png_path, + content_type="image/png" + ) + + try: + storage.connection.make_bucket(current_app.config['MINIO_CAFF_BUCKET_NAME']) + except minio.error.BucketAlreadyOwnedByYou: + pass + storage.connection.fput_object( + current_app.config['MINIO_CAFF_BUCKET_NAME'], + str(item.id), + uploaded_caff_path + ) + + # Always clean up after ourselves + os.unlink(uploaded_caff_path) + os.unlink(converted_png_path) + + db.session.commit() + return redirect(url_for('UploadView:index')) # TODO: report item id