commit 851f45135478e77e59f87e99a75057992234bcac Author: marcsello Date: Tue Nov 23 00:19:29 2021 +0100 initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1df3c9b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,139 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +#Pycharm +.idea/ + +*.md +.gitignore +.git/ +*.yml +contrib/* +postman/* diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..d39867e --- /dev/null +++ b/.drone.yml @@ -0,0 +1,33 @@ +kind: pipeline +type: docker +name: default + +steps: +- name: code-analysis + image: aosapps/drone-sonar-plugin + settings: + sonar_host: + from_secret: SONAR_HOST + sonar_token: + from_secret: SONAR_CODE + +- name: kaniko + image: banzaicloud/drone-kaniko + settings: + registry: registry.kmlabz.com + repo: birbnetes/${DRONE_REPO_NAME} + username: + from_secret: DOCKER_USERNAME + password: + from_secret: DOCKER_PASSWORD + tags: + - latest + - ${DRONE_BUILD_NUMBER} + +- name: ms-teams + image: kuperiu/drone-teams + settings: + webhook: + from_secret: TEAMS_WEBHOOK + when: + status: [ failure ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3327845 --- /dev/null +++ b/.gitignore @@ -0,0 +1,143 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +#Pycharm +.idea/ +*.iml + +# Wing project file +*.wpr + +# Nano, vi, vim lockfile +*.swp + +# log +*.log + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..378ab07 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.9 + +ADD svm_prefilter_service requirements.txt /svm_prefilter_service/ +WORKDIR /svm_prefilter_service/ + +ENV PIP_NO_CACHE_DIR=true +ENV TZ Europe/Budapest +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN pip3 install -r requirements.txt + +ENV GUNICORN_LOGLEVEL="info" + +EXPOSE 8000 +CMD ["gunicorn", "-b", "0.0.0.0:8000", "--log-level", "${GUNICORN_LOGLEVEL}", "app:app"] + diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml new file mode 100644 index 0000000..89f1f43 --- /dev/null +++ b/k8s/configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svm-prefilter-service + labels: + app: svm-prefilter-service + namespace: birbnetes +data: + SENTRY_DSN: "" + RELEASE_ID: kmlabz-k8s + RELEASEMODE: release diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml new file mode 100644 index 0000000..e852dad --- /dev/null +++ b/k8s/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: svm-prefilter-service + namespace: birbnetes + labels: + app: svm-prefilter-service +spec: + replicas: 1 + selector: + matchLabels: + app: svm-prefilter-service + strategy: + type: Recreate + template: + metadata: + labels: + app: svm-prefilter-service + spec: + containers: + - image: registry.kmlabz.com/birbnetes/svm_prefilter_service + imagePullPolicy: Always + name: svm-prefilter-service + envFrom: + - configMapRef: + name: svm-prefilter-service + ports: + - containerPort: 8000 + - name: jaeger-agent + image: jaegertracing/jaeger-agent:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5775 + name: zk-compact-trft + protocol: UDP + - containerPort: 5778 + name: config-rest + protocol: TCP + - containerPort: 6831 + name: jg-compact-trft + protocol: UDP + - containerPort: 6832 + name: jg-binary-trft + protocol: UDP + - containerPort: 14271 + name: admin-http + protocol: TCP + args: + - --reporter.grpc.host-port=dns:///woolsey.tormakristof.eu:14250 + imagePullSecrets: + - name: regcred diff --git a/k8s/service.yaml b/k8s/service.yaml new file mode 100644 index 0000000..ad6a7bd --- /dev/null +++ b/k8s/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: svm-prefilter-service + namespace: birbnetes + labels: + app: svm-prefilter-service +spec: + ports: + - name: svm-prefilter-service + port: 80 + targetPort: 8000 + protocol: TCP + selector: + app: svm-prefilter-service + type: ClusterIP \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..e079f8a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +pytest diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9636571 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,38 @@ +werkzeug~=2.0.1 +requests +blinker +Flask~=2.0.1 +marshmallow~=3.14.1 +Flask-Classful +gunicorn +sentry_sdk +py-healthcheck + +jaeger-client +Flask-Opentracing +opentracing~=2.4.0 + +cython + +six +deprecation + +cycler~=0.10.0 + +eyeD3==0.9.5 +filetype==1.0.6 +hmmlearn==0.2.3 +joblib~=1.0.1 +kiwisolver~=1.2.0 +matplotlib~=3.3.3 +numpy~=1.20.3 +pydub==0.23.1 +pyparsing==2.4.6 +python-dateutil==2.8.1 +scikit-learn~=0.24.0 +scipy~=1.6.2 +simplejson~=3.17.2 + + +pyAudioAnalysis~=0.3.0 +tqdm~=4.61.1 \ No newline at end of file diff --git a/svm_prefilter_service/app.py b/svm_prefilter_service/app.py new file mode 100644 index 0000000..954461a --- /dev/null +++ b/svm_prefilter_service/app.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import sentry_sdk +from sentry_sdk.integrations.flask import FlaskIntegration +from flask import Flask +from werkzeug.middleware.proxy_fix import ProxyFix + +# import stuff +from utils import register_all_error_handlers, register_health_checks + +# import views +from views import FilterView + +from config import Config + +# Tracing stuffs +import jaeger_client +import opentracing +from flask_opentracing import FlaskTracing + +# Setup sentry +if Config.SENTRY_DSN: + sentry_sdk.init( + dsn=Config.SENTRY_DSN, + integrations=[FlaskIntegration()], + traces_sample_rate=0, + send_default_pii=True, + release=Config.RELEASE_ID, + environment=Config.RELEASEMODE + ) + +# create flask app +app = Flask(__name__) +app.config.from_object(Config) +app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1) + +# init stuff + + +# register error handlers +register_all_error_handlers(app) + + +# Setup tracing +def initialize_tracer(): + app.logger.info("Initializing jaeger...") + jaeger_cfg = jaeger_client.Config(config={}, service_name='svm-prefilter-service', validate=True) + tracer = jaeger_cfg.initialize_tracer() + return tracer + + +tracing = FlaskTracing(initialize_tracer, True, app) + +# register views +for view in [FilterView]: + view.register(app, trailing_slash=False) + +register_health_checks(app) + +# start debugging if needed +if __name__ == "__main__": + app.run(debug=True) +else: + import os + + if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""): + import logging + + gunicorn_logger = logging.getLogger('gunicorn.error') + app.logger.handlers = gunicorn_logger.handlers + app.logger.setLevel(gunicorn_logger.level) + + jaeger_logger = logging.getLogger('jaeger_tracing') + jaeger_logger.handlers = gunicorn_logger.handlers + jaeger_logger.setLevel(gunicorn_logger.level) + + app.logger.info("Gunicorn environment detected!") + else: + app.logger.info("Not gunicorn") diff --git a/svm_prefilter_service/config.py b/svm_prefilter_service/config.py new file mode 100644 index 0000000..61197ed --- /dev/null +++ b/svm_prefilter_service/config.py @@ -0,0 +1,9 @@ +import os + + +class Config: + SECRET_KEY = os.environ.get('SECRET_KEY', os.urandom(12)) + + SENTRY_DSN = os.environ.get("SENTRY_DSN") + RELEASE_ID = os.environ.get("RELEASE_ID", "test") + RELEASEMODE = os.environ.get("RELEASEMODE", "dev") diff --git a/svm_prefilter_service/schemas/__init__.py b/svm_prefilter_service/schemas/__init__.py new file mode 100644 index 0000000..e5a0d9b --- /dev/null +++ b/svm_prefilter_service/schemas/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/svm_prefilter_service/tests/__init__.py b/svm_prefilter_service/tests/__init__.py new file mode 100644 index 0000000..21b405d --- /dev/null +++ b/svm_prefilter_service/tests/__init__.py @@ -0,0 +1 @@ +import os diff --git a/svm_prefilter_service/utils/__init__.py b/svm_prefilter_service/utils/__init__.py new file mode 100644 index 0000000..ae22367 --- /dev/null +++ b/svm_prefilter_service/utils/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +from .require_decorators import json_required +from .error_handlers import register_all_error_handlers +from .healthchecks import register_health_checks diff --git a/svm_prefilter_service/utils/error_handlers.py b/svm_prefilter_service/utils/error_handlers.py new file mode 100644 index 0000000..93be2ff --- /dev/null +++ b/svm_prefilter_service/utils/error_handlers.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +from flask import jsonify + + +def get_standard_error_handler(code: int): + def error_handler(err): + return jsonify({"msg": err.description, "status": code}), code + + return error_handler + + +def register_all_error_handlers(app): + """ + function to register all handlers + """ + + error_codes_to_override = [404, 403, 401, 405, 400, 409, 422] + + for code in error_codes_to_override: + app.register_error_handler(code, get_standard_error_handler(code)) diff --git a/svm_prefilter_service/utils/healthchecks.py b/svm_prefilter_service/utils/healthchecks.py new file mode 100644 index 0000000..709a5a5 --- /dev/null +++ b/svm_prefilter_service/utils/healthchecks.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +from healthcheck import HealthCheck +from flask import Flask + + +def dummy_health_check(): + return True, "very good" + + +def register_health_checks(app: Flask): + health = HealthCheck() + health.add_check(dummy_health_check) + app.add_url_rule("/healthz", "healthcheck", view_func=lambda: health.run()) diff --git a/svm_prefilter_service/utils/require_decorators.py b/svm_prefilter_service/utils/require_decorators.py new file mode 100644 index 0000000..95daa39 --- /dev/null +++ b/svm_prefilter_service/utils/require_decorators.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +from flask import request, abort + +from functools import wraps + + +def json_required(f): + @wraps(f) + def call(*args, **kwargs): + + if request.is_json: + return f(*args, **kwargs) + else: + abort(400, "JSON required") + + return call diff --git a/svm_prefilter_service/views/__init__.py b/svm_prefilter_service/views/__init__.py new file mode 100644 index 0000000..17dcda0 --- /dev/null +++ b/svm_prefilter_service/views/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +from .filter_view import FilterView diff --git a/svm_prefilter_service/views/filter_view.py b/svm_prefilter_service/views/filter_view.py new file mode 100644 index 0000000..aa7df06 --- /dev/null +++ b/svm_prefilter_service/views/filter_view.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +from flask import jsonify, request, abort, current_app, Response +from flask_classful import FlaskView +from utils import json_required + + +class FilterView(FlaskView): + + @json_required + def post(self): + data = request.json + + return Response(status=201)