From 479ca0f44d90e22abbf3f4824ba3dfe4db57f086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Tue, 31 Mar 2020 16:51:31 +0200 Subject: [PATCH] throw out marshmallow --- .dockerignore | 2 +- Dockerfile | 2 +- docker-compose.yml | 39 ++++++++++++++++++++++++++++ requirements.txt | 8 +++--- src/app.py | 12 +++------ src/config.py | 7 +++-- src/models.py | 2 +- src/rabbitmqqueue.py | 18 +++++++++++++ src/resources.py | 62 +++++++++++++++++++++----------------------- src/schemas.py | 25 ------------------ 10 files changed, 102 insertions(+), 75 deletions(-) create mode 100644 docker-compose.yml create mode 100644 src/rabbitmqqueue.py delete mode 100644 src/schemas.py diff --git a/.dockerignore b/.dockerignore index 1df3c9b..4d73456 100644 --- a/.dockerignore +++ b/.dockerignore @@ -136,4 +136,4 @@ dmypy.json .git/ *.yml contrib/* -postman/* +postman/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8e04625..f8842b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,4 @@ COPY ./src . EXPOSE 8080 -ENTRYPOINT ["gunicorn", "-b", "0.0.0.0:8080", "--workers", "1", "--threads", "8", "app:app"] +ENTRYPOINT ["gunicorn", "-b", "0.0.0.0:8080", "--workers", "1", "--threads", "1", "app:app"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..81016c2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3' + +networks: + rabbitmq: + external: false + postgres: + external: false + +services: + test-db: + image: postgres:12 + restart: always + ports: + - "127.0.0.1:5432:5432" + environment: + - POSTGRES_USER=input-service + - POSTGRES_PASSWORD=input-service + - POSTGRES_DB=input-service + networks: + - postgres + volumes: + - postres-volume:/var/lib/postgresql/data + + rabbitmq: + image: "rabbitmq:3" + hostname: "test-rabbitmq" + environment: + RABBITMQ_ERLANG_COOKIE: "akjahsfvbkueasnvfjkhsga" + RABBITMQ_DEFAULT_USER: "rabbitmq" + RABBITMQ_DEFAULT_PASS: "rabbitmq" + RABBITMQ_DEFAULT_VHOST: "/" + networks: + - rabbitmq + ports: + - "127.0.0.1:15672:15672" + - "127.0.0.1:5672:5672" + +volumes: + postres-volume: \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8db33f2..a0416e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -sentry_sdk +sentry_sdk[flask] gunicorn -marshmallow Flask Flask-RESTful requests @@ -8,5 +7,6 @@ werkzeug filetype sqlalchemy flask_sqlalchemy -rstr -pika \ No newline at end of file +xeger +pika +psycopg2-binary \ No newline at end of file diff --git a/src/app.py b/src/app.py index df72145..30ed593 100644 --- a/src/app.py +++ b/src/app.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 import logging -import sentry_sdk from flask import Flask from flask_restful import Api +import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration -import pika +import rabbitmqqueue from config import * from db import db -from resources import * +from resources import SampleResource, SampleParameterResource """ Main Flask RESTful API @@ -37,10 +37,6 @@ db.init_app(app) with app.app_context(): db.create_all() -rabbitmq = pika.BlockingConnection(pika.ConnectionParameters(host=RABBITMQ_HOST)) -rabbitmq_channel = rabbitmq.channel() -rabbitmq_channel.queue_declare(RABBITMQ_QUEUE) - formatter = logging.Formatter( fmt="%(asctime)s - %(levelname)s - %(module)s - %(message)s" ) @@ -53,7 +49,7 @@ logger.setLevel(logging.DEBUG) logger.addHandler(handler) api.add_resource(SampleResource, "/v1/input/sample") -api.add_resource(SampleParameterResource, '/v1/input/sample') +api.add_resource(SampleParameterResource, '/v1/input/sample/') if __name__ == "__main__": app.run( diff --git a/src/config.py b/src/config.py index fe0549e..2c036a2 100644 --- a/src/config.py +++ b/src/config.py @@ -18,12 +18,15 @@ DEBUG = os.environ.get("INPUT_SERVICE_DEBUG", True) SENTRY_DSN = os.environ.get("SENTRY_DSN") -RELEASE_ID = os.environ.get("RELEASE_ID") -RELEASEMODE = os.environ.get("INPUT_SERVICE_RELEASEMODE") +RELEASE_ID = os.environ.get("RELEASE_ID", "test") +RELEASEMODE = os.environ.get("INPUT_SERVICE_RELEASEMODE", "dev") RABBITMQ_HOST = os.getenv("INPUT_RABBITMQ_HOSTNAME", "localhost") RABBITMQ_EXCHANGE = os.getenv("INPUT_RABBITMQ_EXCHANGE", "wave-extract") RABBITMQ_QUEUE = os.getenv("INPUT_RABBITMQ_QUEUE", "wave-extract") +RABBITMQ_USERNAME = os.getenv("INPUT_RABBITMQ_USERNAME", "rabbitmq") +RABBITMQ_PASSWORD = os.getenv("INPUT_RABBITMQ_PASSWORD", "rabbitmq") + POSTGRES_HOSTNAME = os.getenv("INPUT_POSTGRES_HOSTNAME", "localhost") POSTGRES_USERNAME = os.getenv("INPUT_POSTGRES_USERNAME", "input-service") POSTGRES_PASSWORD = os.getenv("INPUT_POSTGRES_PASSWORD", "input-service") diff --git a/src/models.py b/src/models.py index f8c5b3d..8a3cf36 100644 --- a/src/models.py +++ b/src/models.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import db +from db import db from sqlalchemy.sql import func """ diff --git a/src/rabbitmqqueue.py b/src/rabbitmqqueue.py new file mode 100644 index 0000000..9f23509 --- /dev/null +++ b/src/rabbitmqqueue.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import pika +from config import * + +""" +Rabbitmq setup +""" + +__author__ = '@tormakris' +__copyright__ = "Copyright 2020, Birbnetes Team" +__module_name__ = "endpoints" +__version__text__ = "1" + +credentials = pika.PlainCredentials(RABBITMQ_USERNAME, RABBITMQ_PASSWORD) +rabbitmq = pika.BlockingConnection(pika.ConnectionParameters(host=RABBITMQ_HOST,credentials=credentials)) +rabbitmq_channel = rabbitmq.channel() +rabbitmq_channel.queue_declare(RABBITMQ_QUEUE) \ No newline at end of file diff --git a/src/resources.py b/src/resources.py index 28c9bd1..ab76da9 100644 --- a/src/resources.py +++ b/src/resources.py @@ -1,15 +1,13 @@ #!/usr/bin/env python3 -from app import rabbitmq_channel import logging -import json -import rstr -from flask_restful import Resource, reqparse -from werkzeug.datastructures import FileStorage +from xeger import Xeger +from flask_restful import Resource +from flask import request import requests import filetype from db import db from models import SampleMetadata -from schemas import * +from rabbitmqqueue import rabbitmq_channel from config import * """ @@ -35,35 +33,33 @@ class SampleResource(Resource): Post request send to the endpoint :return: """ - parse = reqparse.RequestParser() - parse.add_argument('soundFile', type=FileStorage, location='soundFile') - parse.add_argument('description', type=str, location='description') - args = parse.parse_args() - soundFile = args['soundFile'] - description = args['description'] - kind = filetype.guess(soundFile) - if kind.mime != 'wav': + if 'file' not in request.files: + return {"status": "error", "message": "no file found"}, 469 + else: + soundfile = request.files['file'] + + if 'date' not in request.form: + return {"status": "error", "message": "no date found"}, 470 + else: + date = request.form.get("date") + + if 'device_id' not in request.form: + return {"status": "error", "message": "no device_id found"}, 471 + else: + device_id = request.form.get("device_id") + + kind = filetype.guess(soundfile) + if kind is None or kind.mime != 'wav': LOGGER.error( - "Input file was not WAV. Recieved metadata: {}", - description) + f"Input file was not WAV. Recieved metadata: device_id: {device_id}") return {'status': 'error', 'message': 'Input file not WAV.'}, 415 - try: - desc = json.loads(description) - except Exception as e: - LOGGER.exception(e) - return {'status': 'error', - 'message': 'Input JSON could not be parsed'}, 400 - validate_errors = SampleSchema().validate(desc) - if validate_errors: - LOGGER.error( - "Input JSON did not conform to schema. It was: {}", desc) - return {'status': 'error', - 'message': 'Input JSON schema invalid'}, 417 - generated_tag = rstr.xeger(r'^[a-zA-Z]+[0-9a-zA-Z_]*$', 2, 32) + + xeger = Xeger(limit=32) + generated_tag = xeger.xeger(r'^[a-zA-Z]+[0-9a-zA-Z_]*$') record = SampleMetadata( - device_id=desc['device_id'], - device_date=desc['date'], + device_id=device_id, + device_date=date, tag=generated_tag) try: db.session.add(record) @@ -73,7 +69,7 @@ class SampleResource(Resource): 'tag': (None, generated_tag), 'file': ( 'wave.wav', - soundFile, + soundfile, kind.mime)}) rabbitmq_channel.basic_publish( exchange=RABBITMQ_EXCHANGE, @@ -93,7 +89,7 @@ class SampleResource(Resource): Get all stored items :return: """ - samples = SampleMetadata.query.get.all() + samples = SampleMetadata.query.all() return {"status": "ok", "message": samples}, 200 diff --git a/src/schemas.py b/src/schemas.py deleted file mode 100644 index 552c8da..0000000 --- a/src/schemas.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -from marshmallow import Schema, fields - - -""" -Schemas of input objects -""" - - -__author__ = "@tormakris" -__copyright__ = "Copyright 2020, Birbnetes Team" -__module_name__ = "schemas" -__version__text__ = "1" - - -class SampleSchema(Schema): - """ /v1/email/pay - POST - - Parameters: - - date (date) - - device_id (str) - """ - - date = fields.Date(required=True) - device_id = fields.Str(required=True)