Implemented upload

model_service/app.py

@@ -8,7 +8,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix
 # import stuff
 from model import db
 
-from utils import register_all_error_handlers
+from utils import register_all_error_handlers, storage
 
 # import views
 from views import ModelView
@@ -33,7 +33,9 @@ app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI', "sqlite:/
 app.config['MINIO_ENDPOINT'] = os.environ['MINIO_ENDPOINT']
 app.config['MINIO_ACCESS_KEY'] = os.environ['MINIO_ACCESS_KEY']
 app.config['MINIO_SECRET_KEY'] = os.environ['MINIO_SECRET_KEY']
-app.config['MINIO_BUCKET_NAME'] = os.environ['MINIO_BUCKET_NAME']
+app.config['MINIO_MODEL_BUCKET_NAME'] = os.environ['MINIO_MODEL_BUCKET_NAME']
+app.config['MINIO_MEANS_BUCKET_NAME'] = os.environ['MINIO_MEANS_BUCKET_NAME']
+
 
 # important stuff
 app.secret_key = os.environ.get('SECRET_KEY', os.urandom(12))
@@ -42,6 +44,7 @@ app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 
 # initialize stuff
 db.init_app(app)
+storage.init_app(app)
 
 with app.app_context():
     db.create_all()

model_service/model/aimodel.py

@@ -14,5 +14,5 @@ class AIModel(db.Model):
     mid_step = db.Column(db.Float)
     short_window = db.Column(db.Float)
     short_step = db.Column(db.Float)
-    compute_beat = db.Column(db.Float)
+    compute_beat = db.Column(db.Boolean)
     type = db.Column(db.String(15))

model_service/model/default.py

@@ -3,6 +3,7 @@ from .db import db
 from sqlalchemy.dialects.postgresql import UUID
 
 
-class Default(db.AIModel):
+class Default(db.Model):
-    default_id = db.Column(UUID(as_uuid=True), db.ForeignKey("aimodel.id"), nullable=False)
+    id = db.Column(db.Integer, primary_key=True, auto_increment=True)
-    default = db.relationship("AIModel")
+#    aimodel_id = db.Column(UUID(as_uuid=True), db.ForeignKey("AIModel.id"), nullable=False)
+ #   aimodel = db.relationship("AIModel", backref=db.backref("default", lazy=True))

model_service/schemas/__init__.py

@@ -1,3 +1,4 @@
 #!/usr/bin/env python3
 from .default_schema import DefaultSchema
 from .aimodel_schema import AIModelSchema
+from .info_schema import InfoSchema

model_service/schemas/info_schema.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+from marshmallow import fields, Schema
+from marshmallow.utils import get_value, missing
+from marshmallow.validate import OneOf
+
+import uuid
+
+
+class InfoSchema(Schema):
+    id = fields.UUID(default=uuid.uuid4, missing=uuid.uuid4)
+    type = fields.String(validate=OneOf(['svm', 'svm_rbf', 'knn', 'extratrees', 'gradientboosting', 'randomforest']))
+
+    @classmethod  # This threats none values as missing
+    def get_attribute(cls, attr, obj, default):
+        return get_value(attr, obj, default=default) or missing

model_service/views/model_view.py

@@ -1,21 +1,100 @@
 #!/usr/bin/env python3
 import tempfile
+import os
 from flask import request, jsonify, current_app, abort, Response
 from flask_classful import FlaskView, route
 from model import db, Default, AIModel
 from minio.error import BucketAlreadyExists, BucketAlreadyOwnedByYou, ResponseError, NoSuchKey
-from schemas import AIModelSchema, DefaultSchema
+from schemas import AIModelSchema, DefaultSchema, InfoSchema
 from marshmallow.exceptions import ValidationError
 from utils import json_required, storage
+from pyAudioAnalysis.audioTrainTest import load_model, load_model_knn
 
 
 class ModelView(FlaskView):
     aimodel_schema = AIModelSchema(many=False)
     default_schema = DefaultSchema(many=False)
+    info_schema = InfoSchema(many=False)
+
+    def _ensure_buckets(self):
+        for bucket_name in [current_app.config['MINIO_MEANS_BUCKET_NAME'],
+                            current_app.config['MINIO_MODEL_BUCKET_NAME']]:
+            try:
+                storage.connection.make_bucket(bucket_name)
+
+            except BucketAlreadyOwnedByYou as err:
+                pass
+            except BucketAlreadyExists as err:
+                pass
+                # Everything else should be raised
 
-    @json_required
     def post(self):
-        return jsonify({})
+
+        # get important data from the request
+        try:
+            info = self.info_schema.loads(request.form.get('info'))
+        except ValidationError as e:
+            abort(400, str(e))
+
+        # check for conflict
+        m = AIModel.query.filter_by(id=info['id']).first()
+        if m:
+            abort(409)
+
+        # get and validate file
+        model_file = request.files['modelFile']
+
+        if model_file.content_length <= 0:
+            abort(411, f"Content length for modelFile is not a positive integer or missing.")
+
+        means_file = request.files['meansFile']
+
+        if means_file.content_length <= 0:
+            abort(411, f"Content length for meansFile is not a positive integer or missing.")
+
+        # create bucket if necessary
+        self._ensure_buckets()
+
+        # Temporarily save the file, because pyAudioAnalysis can only read files
+        _, temp_model_filename = tempfile.mkstemp()
+        temp_means_filename = temp_model_filename + "MEANS"
+
+        model_file.save(temp_model_filename)
+        means_file.save(temp_means_filename)
+
+        try:
+
+            if info['type'] == 'knn':
+                _, _, _, _, mid_window, mid_step, short_window, short_step, compute_beat \
+                    = load_model_knn(temp_model_filename)
+            else:
+                _, _, _, _, mid_window, mid_step, short_window, short_step, compute_beat \
+                    = load_model(temp_model_filename)
+
+            # Because of pyAudiomeme the files already saved, so we just use the file uploader functions
+            storage.connection.fput_object(
+                current_app.config['MINIO_MODEL_BUCKET_NAME'],
+                str(info['id']),
+                temp_model_filename
+            )
+
+            storage.connection.fput_object(
+                current_app.config['MINIO_MEANS_BUCKET_NAME'],
+                str(info['id']),
+                temp_means_filename
+            )
+
+        finally:
+            os.remove(temp_model_filename)
+            os.remove(temp_means_filename)
+
+        m = AIModel(mid_window=mid_window, mid_step=mid_step, short_window=short_window, short_step=short_step,
+                    compute_beat=compute_beat, type=info['type'])
+
+        db.session.add(m)
+        db.session.commit()
+
+        return jsonify(self.aimodel_schema.dump(m)), 200
 
     def get(self, _id: str):
 
@@ -26,7 +105,7 @@ class ModelView(FlaskView):
             m = AIModel.query.fiter_by(id=_id).first_or_404()
 
         try:
-            data = storage.connection.get_object(current_app.config['MINIO_BUCKET_NAME'], m.id)
+            data = storage.connection.get_object(current_app.config['MINIO_BUCKET_NAME'], str(m.id))
         except NoSuchKey:
             abort(500, "The ID is stored in the database but not int the Object Store")
 
@@ -70,4 +149,4 @@ class ModelView(FlaskView):
         db.session.add(new_default)
         db.session.commit()
 
-        return jsonify(self.aimodel_schema.dump(m))
+        return '', 204