model-service/model_service/views/cnn_view.py

177 lines
6.9 KiB
Python
Raw Normal View History

2020-07-28 20:19:52 +02:00
#!/usr/bin/env python3
from flask import request, jsonify, current_app, abort, Response
from flask_classful import FlaskView, route
from model import db, Default, AIModel, AIModelType
from minio.error import NoSuchKey
from schemas import AIModelSchema, DefaultSchema, InfoSchema
from marshmallow.exceptions import ValidationError
2020-10-02 03:28:40 +02:00
from utils import multipart_required, storage, ensure_buckets
2021-08-05 17:54:11 +02:00
import opentracing
2020-07-28 20:19:52 +02:00
class CNNView(FlaskView):
2020-09-14 02:05:14 +02:00
route_base = 'cnn'
2020-07-28 20:19:52 +02:00
aimodel_schema = AIModelSchema(many=False)
info_schema = InfoSchema(many=False)
2020-10-03 14:42:44 +02:00
MODEL_DIRECTORY = "model/"
WEIGHTS_DIRECTORY = "weights/"
2020-10-02 03:28:40 +02:00
@multipart_required
2020-07-28 20:19:52 +02:00
def post(self):
# get important data from the request
2021-08-05 17:54:11 +02:00
with opentracing.tracer.start_active_span('parseAndValidate'):
try:
info = self.info_schema.loads(request.form.get('info'))
except ValidationError as e:
return abort(400, str(e))
2020-07-28 20:19:52 +02:00
2021-08-05 17:54:11 +02:00
# check for conflict
m = AIModel.query.filter_by(id=info['id']).first()
if m:
return abort(409)
2020-07-28 20:19:52 +02:00
2021-08-05 17:54:11 +02:00
# get and validate file
model_file = request.files['modelFile']
2020-07-28 20:19:52 +02:00
2021-08-05 17:54:11 +02:00
if model_file.content_length <= 0:
return abort(411, f"Content length for modelFile is not a positive integer or missing.")
2020-07-28 20:19:52 +02:00
2021-08-05 17:54:11 +02:00
weights_file = request.files['weightsFile']
2020-07-28 20:19:52 +02:00
2021-08-05 17:54:11 +02:00
if weights_file.content_length <= 0:
return abort(411, f"Content length for weightsFile is not a positive integer or missing.")
2020-07-28 20:19:52 +02:00
# create bucket if necessary
2021-08-05 17:54:11 +02:00
with opentracing.tracer.start_active_span('ensureBuckets'):
ensure_buckets()
2020-07-28 20:19:52 +02:00
2020-09-14 02:25:31 +02:00
# Create the entry in the db
2021-08-05 17:54:11 +02:00
with opentracing.tracer.start_active_span('sqlalchemy.create'):
m = AIModel(id=info['id'], type=AIModelType.cnn, target_class_name=info['target_class_name'])
db.session.add(m)
2020-09-14 02:25:31 +02:00
2020-07-28 20:19:52 +02:00
# Put files into MinIO
2021-08-05 17:54:11 +02:00
with opentracing.tracer.start_active_span('putObjectsInMinio'):
with opentracing.tracer.start_active_span(
'minio.putObject',
tags={
"bucket": current_app.config['MINIO_CNN_BUCKET_NAME'],
"object_name": self.MODEL_DIRECTORY + str(m.id),
"length": model_file.content_length,
"component": "model"
}
):
storage.connection.put_object(
current_app.config['MINIO_CNN_BUCKET_NAME'],
self.MODEL_DIRECTORY + str(m.id),
model_file,
model_file.content_length,
content_type=model_file.content_type
)
with opentracing.tracer.start_active_span(
'minio.putObject',
tags={
"bucket": current_app.config['MINIO_CNN_BUCKET_NAME'],
"object_name": self.WEIGHTS_DIRECTORY + str(m.id),
"length": model_file.content_type,
"component": "weights"
}
):
storage.connection.put_object(
current_app.config['MINIO_CNN_BUCKET_NAME'],
self.WEIGHTS_DIRECTORY + str(m.id),
weights_file,
weights_file.content_length,
content_type=weights_file.content_type
)
with opentracing.tracer.start_active_span('sqlalchemy.commit'):
db.session.commit()
2020-07-28 20:19:52 +02:00
return jsonify(self.aimodel_schema.dump(m)), 200
2020-10-02 03:44:23 +02:00
def delete(self, id_: str):
2020-07-28 20:19:52 +02:00
2021-08-05 17:54:11 +02:00
with opentracing.tracer.start_active_span(
'sqlalchemy.select',
tags={"aimodel_type": AIModelType.cnn, "id": id_}
):
if id_ == "$default":
default = Default.query.filter_by(type=AIModelType.cnn).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.cnn, id=id_).first_or_404()
with opentracing.tracer.start_active_span('removeFromMinio'):
with opentracing.tracer.start_active_span(
'minio.removeObject',
tags={
"bucket": current_app.config['MINIO_CNN_BUCKET_NAME'],
"object_name": self.WEIGHTS_DIRECTORY + str(m.id),
"component": "weights"
}
):
storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'],
self.WEIGHTS_DIRECTORY + str(m.id))
with opentracing.tracer.start_active_span(
'minio.removeObject',
tags={
"bucket": current_app.config['MINIO_CNN_BUCKET_NAME'],
"object_name": self.MODEL_DIRECTORY + str(m.id),
"component": "model"
}
):
storage.connection.remove_object(current_app.config['MINIO_CNN_BUCKET_NAME'],
self.MODEL_DIRECTORY + str(m.id))
with opentracing.tracer.start_active_span(
'sqlalchemy.delete',
tags={"aimodel_type": AIModelType.cnn, "id": id_}
):
db.session.delete(m)
with opentracing.tracer.start_active_span('sqlalchemy.commit'):
db.session.commit()
2020-07-28 20:19:52 +02:00
return '', 204
2020-10-02 03:44:23 +02:00
@route('<id_>/file')
def get_file(self, id_: str):
2020-07-28 20:19:52 +02:00
2021-08-05 17:54:11 +02:00
with opentracing.tracer.start_active_span(
'sqlalchemy.select',
tags={"aimodel_type": AIModelType.cnn, "id": id_}
):
if id_ == "$default":
default = Default.query.filter_by(type=AIModelType.cnn).first_or_404()
m = default.aimodel
else:
m = AIModel.query.filter_by(type=AIModelType.cnn, id=id_).first_or_404()
2020-07-28 20:19:52 +02:00
2020-10-02 03:28:40 +02:00
if "weights" in request.args:
2020-10-03 14:42:44 +02:00
path = self.WEIGHTS_DIRECTORY + str(m.id)
2021-08-05 17:54:11 +02:00
component = "weights"
2020-10-02 03:28:40 +02:00
else:
2020-10-03 14:42:44 +02:00
path = self.MODEL_DIRECTORY + str(m.id)
2021-08-05 17:54:11 +02:00
component = "model"
with opentracing.tracer.start_active_span(
'minio.getObject',
tags={
"bucket": current_app.config['MINIO_CNN_BUCKET_NAME'],
"object_name": path,
"component": component
}
):
# Note: this only initiates the download, the file download itself is streamed
try:
data = storage.connection.get_object(current_app.config['MINIO_CNN_BUCKET_NAME'], path)
except NoSuchKey:
return abort(500, "The ID is stored in the database but not int the Object Store")
2020-07-28 20:19:52 +02:00
2020-10-02 03:28:40 +02:00
return Response(data.stream(), mimetype=data.headers['Content-type'])