Merge pull request #1 from UnstableVortexSecurity/upload

Implemented uploader

requirements.txt

@@ -16,3 +16,5 @@ flask-minio
 bleach
 bcrypt
 flask-mail
+requests
+filemagic

src/app.py

@@ -9,6 +9,7 @@ from flask_mail import Mail
 
 from utils import Config
 from utils import health_database_status, init_security_real_good
+from utils import storage
 from views import ItemView, ProfileView, UploadView, IndexView
 
 from models import db
@@ -42,6 +43,7 @@ db.init_app(app)
 init_security_real_good(app)
 CORS(app)
 Mail(app)
+storage.init_app(app)
 
 for view in [ItemView, ProfileView, UploadView, IndexView]:
     view.register(app, trailing_slash=False)

src/models/item.py

@@ -15,7 +15,7 @@ __version__text__ = "1"
 
 class Item(db.Model):
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
-    name = db.Column(db.String, nullable=False)
+    name = db.Column(db.String(75), nullable=False)
     upload_date = db.Column(db.TIMESTAMP, nullable=False, server_default=func.now())
 
     uploader_id = db.Column(db.Integer, db.ForeignKey("user.id", ondelete="CASCADE"), nullable=False)

src/templates/upload.html

@@ -1,16 +1,24 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% if current_user.is_authenticated %}
-<div class="jumbotron">
+    {% if current_user.is_authenticated %}
+        <div class="jumbotron">
             <h1 class="display-3">File Upload</h1>
-    <br>
+            <div>
                 <form method="POST" action="" enctype="multipart/form-data">
-        <p><input type="file" name="file" accept=".caff"></p>
-        <p><input type="submit" value="Submit" class="btn btn-primary"></p>
+                    <div class="form-group">
+                        <label for="caffTitle">Title: </label>
+                        <input id="caffTitle" name="title" placeholder="My awesome animation!"/>
+                    </div>
+                    <div class="form-group">
+                        <label for="fileChooser">CAFF File: </label>
+                        <input id="fileChooser" type="file" name="file" accept=".caff">
+                    </div>
+                    <button type="submit" class="btn btn-primary">Upload!</button>
                 </form>
-</div>
-{% else %}
-<p><a href="{{ url_for_security('login') }}">Log in</a> to upload an animation.</p>
-{% endif %}
+            </div>
+        </div>
+    {% else %}
+        <p><a href="{{ url_for_security('login') }}">Log in</a> to upload an animation.</p>
+    {% endif %}
 {% endblock %}

src/utils/__init__.py

@@ -1,3 +1,7 @@
 from .healthchecks import health_database_status
 from .security import security, init_security_real_good
 from .config import Config
+from .storage import storage
+from .md5stuffs import calculate_md5_sum_for_file, write_file_from_stream_to_file_like_while_calculating_md5
+from .exceptions import FileIntegrityError
+from .caff_previewer import create_caff_preview

src/utils/caff_previewer.py

@@ -0,0 +1,56 @@
+import tempfile
+import os
+import os.path
+import shutil
+import requests
+from flask import current_app
+from .md5stuffs import calculate_md5_sum_for_file, write_file_from_stream_to_file_like_while_calculating_md5
+from .exceptions import FileIntegrityError
+import magic
+
+EXPECTED_PREVIEW_MIMETYPE = 'image/png'
+
+# The response is validated for
+# - integrity (2-way)
+# - mime type (both header and file identifying)
+# - size (handled by md5sum saver, generator thingy)
+
+def create_caff_preview(src_file: str) -> str:  # hopefully returns a file path
+
+    # Send file for previewing
+    uploaded_caff_md5sum = calculate_md5_sum_for_file(src_file)
+
+    with open(src_file, 'rb') as f:
+        r = requests.post(current_app.config['CAFF_PREVIEWER_ENDPOINT'], data=f, stream=True)
+
+    r.raise_for_status()
+
+    # Verify the results while saving the file
+    if r.headers.get("Content-type") != EXPECTED_PREVIEW_MIMETYPE:
+        raise ValueError(f"Converter output (reported by header) is not {EXPECTED_PREVIEW_MIMETYPE}")
+
+    if r.headers.get("X-request-checksum") != uploaded_caff_md5sum:
+        # This really is the most pointless check in the world
+        # But it was fun to implement
+        raise FileIntegrityError("File sent for previewing and received by previewer differ")
+
+    converted_png_fd, converted_png_path = tempfile.mkstemp(
+        prefix=os.path.basename(src_file).split('.')[0],
+        suffix='.png'
+    )
+
+    with open(converted_png_fd, "wb") as f:
+        converted_png_md5sum = write_file_from_stream_to_file_like_while_calculating_md5(r.raw, f)
+
+    if r.headers.get("X-response-checksum") != converted_png_md5sum:
+        # This does not have much point either
+        raise FileIntegrityError("File sent by previewer and received by the app differ")
+
+    with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as m:
+        calculated_mimetype = m.id_filename(converted_png_path)
+
+    if calculated_mimetype != EXPECTED_PREVIEW_MIMETYPE:
+        raise ValueError(f"Converter output (calculated from file) is not {EXPECTED_PREVIEW_MIMETYPE}")
+
+    del r
+    return converted_png_path

src/utils/config.py

@@ -30,7 +30,17 @@ class Config:
     MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')
     SECURITY_EMAIL_SENDER = os.environ.get('SECURITY_EMAIL_SENDER', 'noreply@unstablevortex.kmlabz.com')
 
+    CAFF_PREVIEWER_ENDPOINT = os.environ["CAFF_PREVIEWER_ENDPOINT"]  # This should prevent application startup
+
+    # Those all should fail the application startup
+    MINIO_ENDPOINT = os.environ["MINIO_ENDPOINT"]
+    MINIO_ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"]
+    MINIO_SECRET_KEY = os.environ["MINIO_SECRET_KEY"]
+    MINIO_SECURE = os.environ.get("MINIO_SECURE", "true").upper() == 'TRUE'
+
     # Some constant configured stuff configs
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     SECURITY_REGISTERABLE = True
     SECURITY_PASSWORD_HASH = "bcrypt"
+    MINIO_PREVIEW_BUCKET_NAME = "previews"
+    MINIO_CAFF_BUCKET_NAME = "caff"

src/utils/exceptions.py

@@ -0,0 +1,4 @@
+
+
+class FileIntegrityError(BaseException):
+    pass

src/utils/md5stuffs.py

@@ -0,0 +1,34 @@
+import hashlib
+
+
+def write_file_from_stream_to_file_like_while_calculating_md5(stream, f, maxsize: int = 536870912,
+                                                              chunksize: int = 4096) -> str:
+    m = hashlib.md5()  # nosec: md5 is used only for integrity checking here
+
+    total_recieved = 0
+
+    # Begin receiving the file
+
+    while True:  # This is where uploading happens
+        chunk = stream.read(chunksize)
+        if len(chunk) == 0:
+            break
+
+        total_recieved += len(chunk)
+        if total_recieved > maxsize:
+            raise OverflowError("File too big")
+
+        m.update(chunk)
+        f.write(chunk)
+
+    return m.hexdigest()
+
+
+def calculate_md5_sum_for_file(fname) -> str:
+    m = hashlib.md5()  # nosec: md5 is used only for integrity checking here
+
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            m.update(chunk)
+
+    return m.hexdigest()

src/utils/storage.py

@@ -0,0 +1,3 @@
+from flask_minio import Minio
+
+storage = Minio()

src/views/uploadview.py

@@ -1,6 +1,17 @@
 #!/usr/bin/env python3
-from flask import render_template
+import tempfile
+import os
+import os.path
+import minio
+from flask import render_template, flash, redirect, url_for, request, current_app
 from flask_classful import FlaskView
+from flask_security import current_user, login_required
+from utils import storage
+import bleach
+from models import db, Item
+
+from utils import create_caff_preview
+from requests import HTTPError
 
 """
 Upload VIEW
@@ -16,3 +27,82 @@ class UploadView(FlaskView):
 
     def index(self):
         return render_template('upload.html')
+
+    @login_required
+    def post(self):
+        title = request.form.get('title')
+        title = title[:Item.name.property.columns[0].type.length]
+        title = bleach.clean(title, tags=[])
+
+        if not title:
+            flash("Title must be filled", "primary")
+            return redirect(url_for('UploadView:index'))
+
+        if 'file' not in request.files:
+            flash("No file provided", "primary")
+            return redirect(url_for('UploadView:index'))
+
+        file = request.files['file']
+        uploaded_caff_fd, uploaded_caff_path = tempfile.mkstemp(prefix=current_user.name, suffix='.caff')
+
+        with open(uploaded_caff_fd, "wb") as f:
+            file.save(f)
+
+        # let the memes begin! -----------------------------------------------------------------------------------------
+        success = False
+        try:
+            converted_png_path = create_caff_preview(uploaded_caff_path)
+            success = os.path.isfile(converted_png_path)  # check if the file really is there
+        except HTTPError as e:
+            if e.response.status_code == 400:
+                flash("Invalid CAFF file", "danger")
+            elif e.response.status_code == 413:
+                flash("CAFF file too large", "danger")
+            else:
+                raise  # Whatever... we'll just check the Sentry alert
+
+        except OverflowError:
+            flash("CAFF file too large", "danger")
+
+        except ValueError:
+            flash("Something went wrong, try again later...", "warning")
+
+        if not success:
+            os.unlink(uploaded_caff_path)
+            return redirect(url_for('UploadView:index'))
+
+        # End of the meme part -----------------------------------------------------------------------------------------
+
+        # Upload everything to minio
+        item = Item(name=title, uploader=current_user)
+        db.session.add(item)
+        db.session.flush()  # To obtain an id
+
+        try:
+            storage.connection.make_bucket(current_app.config['MINIO_PREVIEW_BUCKET_NAME'])
+        except minio.error.BucketAlreadyOwnedByYou:
+            pass
+
+        storage.connection.fput_object(
+            current_app.config['MINIO_PREVIEW_BUCKET_NAME'],
+            str(item.id),
+            converted_png_path,
+            content_type="image/png"
+        )
+
+        try:
+            storage.connection.make_bucket(current_app.config['MINIO_CAFF_BUCKET_NAME'])
+        except minio.error.BucketAlreadyOwnedByYou:
+            pass
+        storage.connection.fput_object(
+            current_app.config['MINIO_CAFF_BUCKET_NAME'],
+            str(item.id),
+            uploaded_caff_path
+        )
+
+        # Always clean up after ourselves
+        os.unlink(uploaded_caff_path)
+        os.unlink(converted_png_path)
+
+        db.session.commit()
+        return redirect(url_for('UploadView:index'))  # TODO: report item id