model-service/model_service/views/svm_view.py

#!/usr/bin/env python3
import tempfile
import os
from flask import request, jsonify, current_app, abort, Response, url_for
from flask_classful import FlaskView, route
from model import db, Default, AIModel, AIModelType, SVMDetails
from minio.error import NoSuchKey
from schemas import AIModelSchema, InfoSchema
from marshmallow.exceptions import ValidationError
from utils import storage, ensure_buckets, multipart_required
from pyAudioAnalysis.audioTrainTest import load_model
import opentracing


class SVMView(FlaskView):
    route_base = 'svm'

    aimodel_schema = AIModelSchema(many=False)
    info_schema = InfoSchema(many=False)

    MODEL_DIRECTORY = "model/"
    MEANS_DIRECTORY = "means/"

    @multipart_required
    def post(self):

        # get important data from the request
        with opentracing.tracer.start_active_span('parseAndValidate'):
            try:
                info = self.info_schema.loads(request.form.get('info'))
            except ValidationError as e:
                return abort(400, str(e))

            # check for conflict
            m = AIModel.query.filter_by(id=info['id']).first()
            if m:
                return abort(409)

            # get and validate file
            model_file = request.files['modelFile']

            if model_file.content_length <= 0:
                return abort(411, f"Content length for modelFile is not a positive integer or missing.")

            means_file = request.files['meansFile']

            if means_file.content_length <= 0:
                return abort(411, f"Content length for meansFile is not a positive integer or missing.")

        # create bucket if necessary
        with opentracing.tracer.start_active_span('ensureBuckets'):
            ensure_buckets()

        # Temporarily save the file, because pyAudioAnalysis can only read files
        with opentracing.tracer.start_active_span('tempfile.save'):
            temp_model_handle, temp_model_filename = tempfile.mkstemp()
            temp_means_filename = temp_model_filename + "MEANS"

            os.close(temp_model_handle)  # BRUUUUH

            model_file.save(temp_model_filename)
            means_file.save(temp_means_filename)

        with opentracing.tracer.start_active_span('pyAudioAnalysis.readModel'):
            try:

                _, _, _, classes, mid_window, mid_step, short_window, short_step, compute_beat \
                    = load_model(temp_model_filename)

                if info['target_class_name'] not in classes:
                    return abort(400, f"This model does not have a class named {info['target_class_name']}")

                # Because of pyAudiomeme the files already saved, so we just use the file uploader functions
                storage.connection.fput_object(
                    current_app.config['MINIO_SVM_BUCKET_NAME'],
                    self.MODEL_DIRECTORY + str(info['id']),
                    temp_model_filename
                )

                storage.connection.fput_object(
                    current_app.config['MINIO_SVM_BUCKET_NAME'],
                    self.MEANS_DIRECTORY + str(info['id']),
                    temp_means_filename
                )

            finally:
                os.remove(temp_model_filename)
                os.remove(temp_means_filename)

        with opentracing.tracer.start_active_span('sqlalchemy.create'):
            m = AIModel(id=info['id'], type=AIModelType.svm, target_class_name=info['target_class_name'])
            db.session.add(m)

            d = SVMDetails(
                aimodel=m,
                mid_window=mid_window,
                mid_step=mid_step,
                short_window=short_window,
                short_step=short_step,
                compute_beat=compute_beat
            )
            db.session.add(d)

        with opentracing.tracer.start_active_span('sqlalchemy.commit'):
            db.session.commit()

        return jsonify(self.aimodel_schema.dump(m)), 200

    def delete(self, id_: str):

        with opentracing.tracer.start_active_span(
                'sqlalchemy.select',
                tags={"aimodel_type": AIModelType.svm, "id": id_}
        ):
            if id_ == "$default":
                default = Default.query.filter_by(type=AIModelType.svm).first_or_404()
                m = default.aimodel
            else:
                m = AIModel.query.filter_by(type=AIModelType.svm, id=id_).first_or_404()

        with opentracing.tracer.start_active_span('removeFromMinio'):
            with opentracing.tracer.start_active_span(
                    'minio.removeObject',
                    tags={
                        "bucket": current_app.config['MINIO_SVM_BUCKET_NAME'],
                        "object_name": self.MEANS_DIRECTORY + str(m.id),
                        "component": "means"
                    }
            ):
                storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'],
                                                 self.MEANS_DIRECTORY + str(m.id))

            with opentracing.tracer.start_active_span(
                    'minio.removeObject',
                    tags={
                        "bucket": current_app.config['MINIO_SVM_BUCKET_NAME'],
                        "object_name": self.MODEL_DIRECTORY + str(m.id),
                        "component": "model"
                    }
            ):
                storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'],
                                                 self.MODEL_DIRECTORY + str(m.id))

        with opentracing.tracer.start_active_span(
                'sqlalchemy.delete',
                tags={"aimodel_type": AIModelType.svm, "id": id_}
        ):
            db.session.delete(m)

        with opentracing.tracer.start_active_span('sqlalchemy.commit'):
            db.session.commit()

        return '', 204

    # builtin file proxy
    @route('<id_>/file')
    def get_file(self, id_: str):

        with opentracing.tracer.start_active_span(
                'sqlalchemy.select',
                tags={"aimodel_type": AIModelType.svm, "id": id_}
        ):
            if id_ == "$default":
                default = Default.query.filter_by(type=AIModelType.svm).first_or_404()
                m = default.aimodel
            else:
                m = AIModel.query.filter_by(type=AIModelType.svm, id=id_).first_or_404()

        if "means" in request.args:
            path = self.MEANS_DIRECTORY + str(m.id)
            component = "means"
        else:
            path = self.MODEL_DIRECTORY + str(m.id)
            component = "model"

        with opentracing.tracer.start_active_span(
                'minio.getObject',
                tags={
                    "bucket": current_app.config['MINIO_SVM_BUCKET_NAME'],
                    "object_name": path,
                    "component": component
                }
        ):
            try:
                data = storage.connection.get_object(current_app.config['MINIO_SVM_BUCKET_NAME'], path)
            except NoSuchKey:
                return abort(500, "The ID is stored in the database but not int the Object Store")

        return Response(data.stream(), mimetype=data.headers['Content-type'])