birbnetes/model-service
birbnetes
/
model-service
4
0
Fork 0
Code Issues Releases Wiki Activity

Revised API endpoints
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Pünkösd Marcell 2020-10-02 03:28:40 +02:00
parent 7650ae2369
commit 38509c5a39
8 changed files with 143 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
model_service/app.py
View File

@ -11,7 +11,7 @@ from model import db
from utils import register_all_error_handlers, storage
# import views
from views import SVMView, CNNView
from views import SVMView, CNNView, RootView
# Setup sentry
SENTRY_DSN = os.environ.get("SENTRY_DSN")
@ -56,7 +56,7 @@ def create_db():
register_all_error_handlers(app)
# register views
for view in [SVMView, CNNView]:
for view in [SVMView, CNNView, RootView]:
view.register(app, trailing_slash=False, route_prefix='/model')
# start debuggig if needed

5
model_service/model/aimodel.py
View File

@ -7,8 +7,9 @@ import enum
class AIModelType(enum.Enum):
SVM = 1
CNN = 2
# Optimally this would be upper case (as a convention for enums) But we want this to be the same as in the url schema
svm = 1
cnn = 2
class AIModel(db.Model):

2
model_service/utils/__init__.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
from .require_decorators import json_required
from .require_decorators import json_required, multipart_required
from .error_handlers import register_all_error_handlers
from .storage import storage, ensure_buckets

12
model_service/utils/require_decorators.py
View File

@ -14,3 +14,15 @@ def json_required(f):
abort(400, "JSON required")
return call
def multipart_required(f):
@wraps(f)
def call(*args, **kwargs):
if request.form:
return f(*args, **kwargs)
else:
abort(400, "multipart/form-data required")
return call

3
model_service/views/__init__.py
View File

@ -1,3 +1,4 @@
#!/usr/bin/env python3
from .svm_view import SVMView
from .cnn_view import CNNView
from .cnn_view import CNNView
from .root_view import RootView

82
model_service/views/cnn_view.py
View File

@ -5,21 +5,16 @@ from model import db, Default, AIModel, AIModelType
from minio.error import NoSuchKey
from schemas import AIModelSchema, DefaultSchema, InfoSchema
from marshmallow.exceptions import ValidationError
from utils import json_required, storage, ensure_buckets
from utils import multipart_required, storage, ensure_buckets
class CNNView(FlaskView):
route_base = 'cnn'
aimodel_schema = AIModelSchema(many=False)
aimodels_schema = AIModelSchema(many=True, exclude=['timestamp', 'details'])
default_schema = DefaultSchema(many=False)
info_schema = InfoSchema(many=False)
def index(self):
models = AIModel.query.filter_by(type=AIModelType.CNN).all()
return jsonify(self.aimodels_schema.dump(models)), 200
@multipart_required
def post(self):
# get important data from the request
@ -48,7 +43,7 @@ class CNNView(FlaskView):
ensure_buckets()
# Create the entry in the db
m = AIModel(id=info['id'], type=AIModelType.CNN, target_class_name=info['target_class_name'])
m = AIModel(id=info['id'], type=AIModelType.cnn, target_class_name=info['target_class_name'])
# Put files into MinIO
storage.connection.put_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "model/" + str(m.id), model_file,
@ -62,14 +57,30 @@ class CNNView(FlaskView):
return jsonify(self.aimodel_schema.dump(m)), 200
def get(self, _id: str):
def delete(self, _id: str):
if _id == "$default":
# TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza
default = Default.query.filter_by(type=AIModelType.CNN).first_or_404()
default = Default.query.filter_by(type=AIModelType.cnn).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.CNN, id=_id).first_or_404()
m = AIModel.query.filter_by(type=AIModelType.cnn, id=_id).first_or_404()
storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "weights/" + str(m.id))
storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "model/" + str(m.id))
db.session.delete(m)
db.session.commit()
return '', 204
@route('<_id>/file')
def get_file(self, _id: str):
if _id == "$default":
default = Default.query.filter_by(type=AIModelType.cnn).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.cnn, id=_id).first_or_404()
if "weights" in request.args:
path = "weights/" + str(m.id)
@ -82,50 +93,3 @@ class CNNView(FlaskView):
abort(500, "The ID is stored in the database but not int the Object Store")
return Response(data.stream(), mimetype=data.headers['Content-type'])
@route('<_id>/details')
def get_details(self, _id: str):
if _id == "$default":
# TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza
default = Default.query.filter_by(type=AIModelType.CNN).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.CNN, id=_id).first_or_404()
return jsonify(self.aimodel_schema.dump(m))
def delete(self, _id: str):
if _id == "$default":
# TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza
default = Default.query.filter_by(type=AIModelType.CNN).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.CNN, id=_id).first_or_404()
storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "weights/" + str(m.id))
storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'], "model/" + str(m.id))
db.session.delete(m)
db.session.commit()
return '', 204
@json_required
@route('$default', methods=['PUT'])
def put_default(self):
try:
req = self.default_schema.load(request.json)
except ValidationError as e:
abort(400, str(e))
m = AIModel.query.filter_by(type=AIModelType.CNN, id=req['id']).first_or_404()
Default.query.filter_by(type=AIModelType.CNN).delete()
new_default = Default(type=AIModelType.CNN, aimodel=m)
db.session.add(new_default)
db.session.commit()
return '', 204

73
model_service/views/root_view.py Normal file
View File

@ -0,0 +1,73 @@
#!/usr/bin/env python3
from flask import jsonify, abort, request
from flask_classful import FlaskView, route
from marshmallow import ValidationError
from utils import json_required
from model import db, AIModel, AIModelType, Default
from schemas import AIModelSchema, DefaultSchema
class RootView(FlaskView):
route_base = '/'
aimodels_schema = AIModelSchema(many=True, exclude=['timestamp', 'details', 'target_class_name'])
aimodel_schema = AIModelSchema(many=False)
default_schema = DefaultSchema(many=False)
## Shared stuff goes here
def index(self):
models = AIModel.query.all()
return jsonify(self.aimodels_schema.dump(models))
@route('/<type_>')
def get_models(self, type_: str):
try:
aimodel_type = AIModelType[type_]
except KeyError:
abort(404, "Unknown type")
models = AIModel.query.filter_by(type=aimodel_type).all()
return jsonify(self.aimodels_schema.dump(models)), 200
@route('/<type_>/<id_>')
def get_model(self, type_: str, id_: str):
try:
aimodel_type = AIModelType[type_]
except KeyError:
abort(404, "Unknown type")
if id_ == "$default":
default = Default.query.filter_by(type=aimodel_type).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=aimodel_type, id=id_).first_or_404()
return jsonify(self.aimodel_schema.dump(m))
@json_required
@route('/<type_>/$default', methods=['PUT'])
def put_default(self, type_: str):
try:
aimodel_type = AIModelType[type_]
except KeyError:
abort(404, "Unknown type")
try:
req = self.default_schema.load(request.json)
except ValidationError as e:
abort(400, str(e))
m = AIModel.query.filter_by(type=aimodel_type, id=req['id']).first_or_404()
Default.query.filter_by(type=aimodel_type).delete()
new_default = Default(type=aimodel_type, aimodel=m)
db.session.add(new_default)
db.session.commit()
return '', 204

89
model_service/views/svm_view.py
View File

@ -1,28 +1,23 @@
#!/usr/bin/env python3
import tempfile
import os
from flask import request, jsonify, current_app, abort, Response
from flask import request, jsonify, current_app, abort, Response, url_for
from flask_classful import FlaskView, route
from model import db, Default, AIModel, AIModelType, SVMDetails
from minio.error import NoSuchKey
from schemas import AIModelSchema, DefaultSchema, InfoSchema
from schemas import AIModelSchema, InfoSchema
from marshmallow.exceptions import ValidationError
from utils import json_required, storage, ensure_buckets
from pyAudioAnalysis.audioTrainTest import load_model, load_model_knn
from utils import storage, ensure_buckets, multipart_required
from pyAudioAnalysis.audioTrainTest import load_model
class SVMView(FlaskView):
route_base = 'svm'
aimodel_schema = AIModelSchema(many=False)
aimodels_schema = AIModelSchema(many=True, exclude=['timestamp', 'details'])
default_schema = DefaultSchema(many=False)
info_schema = InfoSchema(many=False)
def index(self):
models = AIModel.query.filter_by(type=AIModelType.SVM).all()
return jsonify(self.aimodels_schema.dump(models)), 200
@multipart_required
def post(self):
# get important data from the request
@ -84,7 +79,7 @@ class SVMView(FlaskView):
os.remove(temp_model_filename)
os.remove(temp_means_filename)
m = AIModel(id=info['id'], type=AIModelType.SVM, target_class_name=info['target_class_name'])
m = AIModel(id=info['id'], type=AIModelType.svm, target_class_name=info['target_class_name'])
d = SVMDetails(
aimodel=m,
@ -101,14 +96,31 @@ class SVMView(FlaskView):
return jsonify(self.aimodel_schema.dump(m)), 200
def get(self, _id: str):
def delete(self, _id: str):
if _id == "$default":
# TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza
default = Default.query.filter_by(type=AIModelType.SVM).first_or_404()
default = Default.query.filter_by(type=AIModelType.svm).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.SVM, id=_id).first_or_404()
m = AIModel.query.filter_by(type=AIModelType.svm, id=_id).first_or_404()
storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "means/" + str(m.id))
storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "model/" + str(m.id))
db.session.delete(m)
db.session.commit()
return '', 204
# builtin file proxy
@route('<_id>/file')
def get_file(self, _id: str):
if _id == "$default":
default = Default.query.filter_by(type=AIModelType.svm).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.svm, id=_id).first_or_404()
if "means" in request.args:
path = "means/" + str(m.id)
@ -121,50 +133,3 @@ class SVMView(FlaskView):
abort(500, "The ID is stored in the database but not int the Object Store")
return Response(data.stream(), mimetype=data.headers['Content-type'])
@route('<_id>/details')
def get_details(self, _id: str):
if _id == "$default":
# TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza
default = Default.query.filter_by(type=AIModelType.SVM).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.SVM, id=_id).first_or_404()
return jsonify(self.aimodel_schema.dump(m))
def delete(self, _id: str):
if _id == "$default":
# TODO: Kitalálni, hogy inkább a latestestest-el térjen-e vissza
default = Default.query.filter_by(type=AIModelType.SVM).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.SVM, id=_id).first_or_404()
storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "means/" + str(m.id))
storage.connection.remove_object(current_app.config['MINIO_SVM_BUCKET_NAME'], "model/" + str(m.id))
db.session.delete(m)
db.session.commit()
return '', 204
@json_required
@route('$default', methods=['PUT'])
def put_default(self):
try:
req = self.default_schema.load(request.json)
except ValidationError as e:
abort(400, str(e))
m = AIModel.query.filter_by(type=AIModelType.SVM, id=req['id']).first_or_404()
Default.query.filter_by(type=AIModelType.SVM).delete()
new_default = Default(type=AIModelType.SVM, aimodel=m)
db.session.add(new_default)
db.session.commit()
return '', 204