From 38509c5a395814126f0f1be6f12d123bfa7ca7f4 Mon Sep 17 00:00:00 2001 From: marcsello Date: Fri, 2 Oct 2020 03:28:40 +0200 Subject: [PATCH] Revised API endpoints --- model_service/app.py | 4 +- model_service/model/aimodel.py | 5 +- model_service/utils/__init__.py | 2 +- model_service/utils/require_decorators.py | 12 +++ model_service/views/__init__.py | 3 +- model_service/views/cnn_view.py | 82 ++++++--------------- model_service/views/root_view.py | 73 +++++++++++++++++++ model_service/views/svm_view.py | 89 +++++++---------------- 8 files changed, 143 insertions(+), 127 deletions(-) create mode 100644 model_service/views/root_view.py diff --git a/model_service/app.py b/model_service/app.py index 3822d87..5960ae3 100644 --- a/model_service/app.py +++ b/model_service/app.py @@ -11,7 +11,7 @@ from model import db from utils import register_all_error_handlers, storage # import views -from views import SVMView, CNNView +from views import SVMView, CNNView, RootView # Setup sentry SENTRY_DSN = os.environ.get("SENTRY_DSN") @@ -56,7 +56,7 @@ def create_db(): register_all_error_handlers(app) # register views -for view in [SVMView, CNNView]: +for view in [SVMView, CNNView, RootView]: view.register(app, trailing_slash=False, route_prefix='/model') # start debuggig if needed diff --git a/model_service/model/aimodel.py b/model_service/model/aimodel.py index e6d60b8..d98ef00 100644 --- a/model_service/model/aimodel.py +++ b/model_service/model/aimodel.py @@ -7,8 +7,9 @@ import enum class AIModelType(enum.Enum): - SVM = 1 - CNN = 2 + # Optimally this would be upper case (as a convention for enums) But we want this to be the same as in the url schema + svm = 1 + cnn = 2 class AIModel(db.Model): diff --git a/model_service/utils/__init__.py b/model_service/utils/__init__.py index 424fba4..fe6829d 100644 --- a/model_service/utils/__init__.py +++ b/model_service/utils/__init__.py @@ -1,4 +1,4 @@ #!/usr/bin/env python3 -from .require_decorators import json_required +from .require_decorators import json_required, multipart_required from .error_handlers import register_all_error_handlers from .storage import storage, ensure_buckets \ No newline at end of file diff --git a/model_service/utils/require_decorators.py b/model_service/utils/require_decorators.py index ba5b9ab..4d5db35 100644 --- a/model_service/utils/require_decorators.py +++ b/model_service/utils/require_decorators.py @@ -14,3 +14,15 @@ def json_required(f): abort(400, "JSON required") return call + + +def multipart_required(f): + @wraps(f) + def call(*args, **kwargs): + + if request.form: + return f(*args, **kwargs) + else: + abort(400, "multipart/form-data required") + + return call diff --git a/model_service/views/__init__.py b/model_service/views/__init__.py index 69db186..31052cf 100644 --- a/model_service/views/__init__.py +++ b/model_service/views/__init__.py @@ -1,3 +1,4 @@ #!/usr/bin/env python3 from .svm_view import SVMView -from .cnn_view import CNNView \ No newline at end of file +from .cnn_view import CNNView +from .root_view import RootView \ No newline at end of file diff --git a/model_service/views/cnn_view.py b/model_service/views/cnn_view.py index 67d4e82..de4687f 100644 --- a/model_service/views/cnn_view.py +++ b/model_service/views/cnn_view.py @@ -5,21 +5,16 @@ from model import db, Default, AIModel, AIModelType from minio.error import NoSuchKey from schemas import AIModelSchema, DefaultSchema, InfoSchema from marshmallow.exceptions import ValidationError -from utils import json_required, storage, ensure_buckets +from utils import multipart_required, storage, ensure_buckets class CNNView(FlaskView): route_base = 'cnn' aimodel_schema = AIModelSchema(many=False) - aimodels_schema = AIModelSchema(many=True, exclude=['timestamp', 'details']) - default_schema = DefaultSchema(many=False) info_schema = InfoSchema(many=False) - def index(self): - models = AIModel.query.filter_by(type=AIModelType.CNN).all() - return jsonify(self.aimodels_schema.dump(models)), 200 - + @multipart_required def post(self): # get important data from the request @@ -48,7 +43,7 @@ class CNNView(FlaskView): ensure_buckets() # Create the entry in the db - m = AIModel(id=info['id'], type=AIModelType.CNN, target_class_name=info['target_class_name']) + m = AIModel(id=info['id'], type=AIModelType.cnn, target_class_name=info['target_class_name']) # Put files into MinIO storage.connection.put_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "model/" + str(m.id), model_file, @@ -62,14 +57,30 @@ class CNNView(FlaskView): return jsonify(self.aimodel_schema.dump(m)), 200 - def get(self, _id: str): + def delete(self, _id: str): if _id == "$default": - # TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza - default = Default.query.filter_by(type=AIModelType.CNN).first_or_404() + default = Default.query.filter_by(type=AIModelType.cnn).first_or_404() m = default.aimodel else: - m = AIModel.query.filter_by(type=AIModelType.CNN, id=_id).first_or_404() + m = AIModel.query.filter_by(type=AIModelType.cnn, id=_id).first_or_404() + + storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "weights/" + str(m.id)) + storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "model/" + str(m.id)) + + db.session.delete(m) + db.session.commit() + + return '', 204 + + @route('<_id>/file') + def get_file(self, _id: str): + + if _id == "$default": + default = Default.query.filter_by(type=AIModelType.cnn).first_or_404() + m = default.aimodel + else: + m = AIModel.query.filter_by(type=AIModelType.cnn, id=_id).first_or_404() if "weights" in request.args: path = "weights/" + str(m.id) @@ -82,50 +93,3 @@ class CNNView(FlaskView): abort(500, "The ID is stored in the database but not int the Object Store") return Response(data.stream(), mimetype=data.headers['Content-type']) - - @route('<_id>/details') - def get_details(self, _id: str): - - if _id == "$default": - # TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza - default = Default.query.filter_by(type=AIModelType.CNN).first_or_404() - m = default.aimodel - else: - m = AIModel.query.filter_by(type=AIModelType.CNN, id=_id).first_or_404() - - return jsonify(self.aimodel_schema.dump(m)) - - def delete(self, _id: str): - - if _id == "$default": - # TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza - default = Default.query.filter_by(type=AIModelType.CNN).first_or_404() - m = default.aimodel - else: - m = AIModel.query.filter_by(type=AIModelType.CNN, id=_id).first_or_404() - - storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "weights/" + str(m.id)) - storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "model/" + str(m.id)) - - db.session.delete(m) - db.session.commit() - - return '', 204 - - @json_required - @route('$default', methods=['PUT']) - def put_default(self): - - try: - req = self.default_schema.load(request.json) - except ValidationError as e: - abort(400, str(e)) - - m = AIModel.query.filter_by(type=AIModelType.CNN, id=req['id']).first_or_404() - - Default.query.filter_by(type=AIModelType.CNN).delete() - new_default = Default(type=AIModelType.CNN, aimodel=m) - db.session.add(new_default) - db.session.commit() - - return '', 204 diff --git a/model_service/views/root_view.py b/model_service/views/root_view.py new file mode 100644 index 0000000..e9aa189 --- /dev/null +++ b/model_service/views/root_view.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +from flask import jsonify, abort, request +from flask_classful import FlaskView, route +from marshmallow import ValidationError + +from utils import json_required + +from model import db, AIModel, AIModelType, Default + +from schemas import AIModelSchema, DefaultSchema + + +class RootView(FlaskView): + route_base = '/' + + aimodels_schema = AIModelSchema(many=True, exclude=['timestamp', 'details', 'target_class_name']) + aimodel_schema = AIModelSchema(many=False) + + default_schema = DefaultSchema(many=False) + + ## Shared stuff goes here + + def index(self): + models = AIModel.query.all() + return jsonify(self.aimodels_schema.dump(models)) + + @route('/') + def get_models(self, type_: str): + try: + aimodel_type = AIModelType[type_] + except KeyError: + abort(404, "Unknown type") + + models = AIModel.query.filter_by(type=aimodel_type).all() + return jsonify(self.aimodels_schema.dump(models)), 200 + + @route('//') + def get_model(self, type_: str, id_: str): + try: + aimodel_type = AIModelType[type_] + except KeyError: + abort(404, "Unknown type") + + if id_ == "$default": + default = Default.query.filter_by(type=aimodel_type).first_or_404() + m = default.aimodel + else: + m = AIModel.query.filter_by(type=aimodel_type, id=id_).first_or_404() + + return jsonify(self.aimodel_schema.dump(m)) + + @json_required + @route('//$default', methods=['PUT']) + def put_default(self, type_: str): + + try: + aimodel_type = AIModelType[type_] + except KeyError: + abort(404, "Unknown type") + + try: + req = self.default_schema.load(request.json) + except ValidationError as e: + abort(400, str(e)) + + m = AIModel.query.filter_by(type=aimodel_type, id=req['id']).first_or_404() + + Default.query.filter_by(type=aimodel_type).delete() + new_default = Default(type=aimodel_type, aimodel=m) + db.session.add(new_default) + db.session.commit() + + return '', 204 diff --git a/model_service/views/svm_view.py b/model_service/views/svm_view.py index 8c3447a..03a6647 100644 --- a/model_service/views/svm_view.py +++ b/model_service/views/svm_view.py @@ -1,28 +1,23 @@ #!/usr/bin/env python3 import tempfile import os -from flask import request, jsonify, current_app, abort, Response +from flask import request, jsonify, current_app, abort, Response, url_for from flask_classful import FlaskView, route from model import db, Default, AIModel, AIModelType, SVMDetails from minio.error import NoSuchKey -from schemas import AIModelSchema, DefaultSchema, InfoSchema +from schemas import AIModelSchema, InfoSchema from marshmallow.exceptions import ValidationError -from utils import json_required, storage, ensure_buckets -from pyAudioAnalysis.audioTrainTest import load_model, load_model_knn +from utils import storage, ensure_buckets, multipart_required +from pyAudioAnalysis.audioTrainTest import load_model class SVMView(FlaskView): route_base = 'svm' aimodel_schema = AIModelSchema(many=False) - aimodels_schema = AIModelSchema(many=True, exclude=['timestamp', 'details']) - default_schema = DefaultSchema(many=False) info_schema = InfoSchema(many=False) - def index(self): - models = AIModel.query.filter_by(type=AIModelType.SVM).all() - return jsonify(self.aimodels_schema.dump(models)), 200 - + @multipart_required def post(self): # get important data from the request @@ -84,7 +79,7 @@ class SVMView(FlaskView): os.remove(temp_model_filename) os.remove(temp_means_filename) - m = AIModel(id=info['id'], type=AIModelType.SVM, target_class_name=info['target_class_name']) + m = AIModel(id=info['id'], type=AIModelType.svm, target_class_name=info['target_class_name']) d = SVMDetails( aimodel=m, @@ -101,14 +96,31 @@ class SVMView(FlaskView): return jsonify(self.aimodel_schema.dump(m)), 200 - def get(self, _id: str): + def delete(self, _id: str): if _id == "$default": - # TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza - default = Default.query.filter_by(type=AIModelType.SVM).first_or_404() + default = Default.query.filter_by(type=AIModelType.svm).first_or_404() m = default.aimodel else: - m = AIModel.query.filter_by(type=AIModelType.SVM, id=_id).first_or_404() + m = AIModel.query.filter_by(type=AIModelType.svm, id=_id).first_or_404() + + storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "means/" + str(m.id)) + storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "model/" + str(m.id)) + + db.session.delete(m) + db.session.commit() + + return '', 204 + + # builtin file proxy + @route('<_id>/file') + def get_file(self, _id: str): + + if _id == "$default": + default = Default.query.filter_by(type=AIModelType.svm).first_or_404() + m = default.aimodel + else: + m = AIModel.query.filter_by(type=AIModelType.svm, id=_id).first_or_404() if "means" in request.args: path = "means/" + str(m.id) @@ -121,50 +133,3 @@ class SVMView(FlaskView): abort(500, "The ID is stored in the database but not int the Object Store") return Response(data.stream(), mimetype=data.headers['Content-type']) - - @route('<_id>/details') - def get_details(self, _id: str): - - if _id == "$default": - # TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza - default = Default.query.filter_by(type=AIModelType.SVM).first_or_404() - m = default.aimodel - else: - m = AIModel.query.filter_by(type=AIModelType.SVM, id=_id).first_or_404() - - return jsonify(self.aimodel_schema.dump(m)) - - def delete(self, _id: str): - - if _id == "$default": - # TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza - default = Default.query.filter_by(type=AIModelType.SVM).first_or_404() - m = default.aimodel - else: - m = AIModel.query.filter_by(type=AIModelType.SVM, id=_id).first_or_404() - - storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "means/" + str(m.id)) - storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "model/" + str(m.id)) - - db.session.delete(m) - db.session.commit() - - return '', 204 - - @json_required - @route('$default', methods=['PUT']) - def put_default(self): - - try: - req = self.default_schema.load(request.json) - except ValidationError as e: - abort(400, str(e)) - - m = AIModel.query.filter_by(type=AIModelType.SVM, id=req['id']).first_or_404() - - Default.query.filter_by(type=AIModelType.SVM).delete() - new_default = Default(type=AIModelType.SVM, aimodel=m) - db.session.add(new_default) - db.session.commit() - - return '', 204