initial commit

.dockerignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+#Pycharm
+.idea/
+
+*.md
+.gitignore
+.git/
+*.yml
+contrib/*
+postman/*

.drone.yml

@@ -0,0 +1,33 @@
+kind: pipeline
+type: docker
+name: default
+
+steps:
+- name: code-analysis
+  image: aosapps/drone-sonar-plugin
+  settings:
+    sonar_host:
+      from_secret: SONAR_HOST
+    sonar_token:
+      from_secret: SONAR_CODE
+
+- name: kaniko
+  image: banzaicloud/drone-kaniko
+  settings:
+    registry: registry.kmlabz.com
+    repo: birbnetes/${DRONE_REPO_NAME}
+    username:
+      from_secret: DOCKER_USERNAME
+    password:
+      from_secret: DOCKER_PASSWORD
+    tags:
+      - latest
+      - ${DRONE_BUILD_NUMBER}
+
+- name: ms-teams
+  image: kuperiu/drone-teams
+  settings:
+    webhook: 
+      from_secret: TEAMS_WEBHOOK
+  when:
+    status: [ failure ]

.gitignore

@@ -0,0 +1,143 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+#Pycharm
+.idea/
+*.iml
+
+# Wing project file
+*.wpr
+
+# Nano, vi, vim lockfile
+*.swp
+
+# log
+*.log
+

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.9
+
+ADD svm_prefilter_service requirements.txt /svm_prefilter_service/
+WORKDIR /svm_prefilter_service/
+
+ENV PIP_NO_CACHE_DIR=true
+ENV TZ Europe/Budapest
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN pip3 install -r requirements.txt
+
+ENV GUNICORN_LOGLEVEL="info"
+
+EXPOSE 8000
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "--log-level", "${GUNICORN_LOGLEVEL}", "app:app"]
+

k8s/configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svm-prefilter-service
+  labels:
+    app: svm-prefilter-service
+  namespace: birbnetes
+data:
+  SENTRY_DSN: ""
+  RELEASE_ID: kmlabz-k8s
+  RELEASEMODE: release

k8s/deployment.yaml

@@ -0,0 +1,51 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: svm-prefilter-service
+  namespace: birbnetes
+  labels:
+    app: svm-prefilter-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: svm-prefilter-service
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: svm-prefilter-service
+    spec:
+      containers:
+      - image: registry.kmlabz.com/birbnetes/svm_prefilter_service
+        imagePullPolicy: Always
+        name: svm-prefilter-service
+        envFrom:
+          - configMapRef:
+              name: svm-prefilter-service
+        ports:
+        - containerPort: 8000
+      - name: jaeger-agent
+        image: jaegertracing/jaeger-agent:latest
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 5775
+          name: zk-compact-trft
+          protocol: UDP
+        - containerPort: 5778
+          name: config-rest
+          protocol: TCP
+        - containerPort: 6831
+          name: jg-compact-trft
+          protocol: UDP
+        - containerPort: 6832
+          name: jg-binary-trft
+          protocol: UDP
+        - containerPort: 14271
+          name: admin-http
+          protocol: TCP
+        args:
+          - --reporter.grpc.host-port=dns:///woolsey.tormakristof.eu:14250
+      imagePullSecrets:
+      - name: regcred

k8s/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: svm-prefilter-service
+  namespace: birbnetes
+  labels:
+    app: svm-prefilter-service
+spec:
+  ports:
+  - name: svm-prefilter-service
+    port: 80
+    targetPort: 8000
+    protocol: TCP
+  selector:
+    app: svm-prefilter-service
+  type: ClusterIP

requirements-dev.txt

@@ -0,0 +1 @@
+pytest

requirements.txt

@@ -0,0 +1,38 @@
+werkzeug~=2.0.1
+requests
+blinker
+Flask~=2.0.1
+marshmallow~=3.14.1
+Flask-Classful
+gunicorn
+sentry_sdk
+py-healthcheck
+
+jaeger-client
+Flask-Opentracing
+opentracing~=2.4.0
+
+cython
+
+six
+deprecation
+
+cycler~=0.10.0
+
+eyeD3==0.9.5
+filetype==1.0.6
+hmmlearn==0.2.3
+joblib~=1.0.1
+kiwisolver~=1.2.0
+matplotlib~=3.3.3
+numpy~=1.20.3
+pydub==0.23.1
+pyparsing==2.4.6
+python-dateutil==2.8.1
+scikit-learn~=0.24.0
+scipy~=1.6.2
+simplejson~=3.17.2
+
+
+pyAudioAnalysis~=0.3.0
+tqdm~=4.61.1

svm_prefilter_service/app.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+from flask import Flask
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+# import stuff
+from utils import register_all_error_handlers, register_health_checks
+
+# import views
+from views import FilterView
+
+from config import Config
+
+# Tracing stuffs
+import jaeger_client
+import opentracing
+from flask_opentracing import FlaskTracing
+
+# Setup sentry
+if Config.SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=Config.SENTRY_DSN,
+        integrations=[FlaskIntegration()],
+        traces_sample_rate=0,
+        send_default_pii=True,
+        release=Config.RELEASE_ID,
+        environment=Config.RELEASEMODE
+    )
+
+# create flask app
+app = Flask(__name__)
+app.config.from_object(Config)
+app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
+
+# init stuff
+
+
+# register error handlers
+register_all_error_handlers(app)
+
+
+# Setup tracing
+def initialize_tracer():
+    app.logger.info("Initializing jaeger...")
+    jaeger_cfg = jaeger_client.Config(config={}, service_name='svm-prefilter-service', validate=True)
+    tracer = jaeger_cfg.initialize_tracer()
+    return tracer
+
+
+tracing = FlaskTracing(initialize_tracer, True, app)
+
+# register views
+for view in [FilterView]:
+    view.register(app, trailing_slash=False)
+
+register_health_checks(app)
+
+# start debugging if needed
+if __name__ == "__main__":
+    app.run(debug=True)
+else:
+    import os
+
+    if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
+        import logging
+
+        gunicorn_logger = logging.getLogger('gunicorn.error')
+        app.logger.handlers = gunicorn_logger.handlers
+        app.logger.setLevel(gunicorn_logger.level)
+
+        jaeger_logger = logging.getLogger('jaeger_tracing')
+        jaeger_logger.handlers = gunicorn_logger.handlers
+        jaeger_logger.setLevel(gunicorn_logger.level)
+
+        app.logger.info("Gunicorn environment detected!")
+    else:
+        app.logger.info("Not gunicorn")

svm_prefilter_service/config.py

@@ -0,0 +1,9 @@
+import os
+
+
+class Config:
+    SECRET_KEY = os.environ.get('SECRET_KEY', os.urandom(12))
+
+    SENTRY_DSN = os.environ.get("SENTRY_DSN")
+    RELEASE_ID = os.environ.get("RELEASE_ID", "test")
+    RELEASEMODE = os.environ.get("RELEASEMODE", "dev")

svm_prefilter_service/schemas/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

svm_prefilter_service/tests/__init__.py

@@ -0,0 +1 @@
+import os

svm_prefilter_service/utils/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+from .require_decorators import json_required
+from .error_handlers import register_all_error_handlers
+from .healthchecks import register_health_checks

svm_prefilter_service/utils/error_handlers.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+from flask import jsonify
+
+
+def get_standard_error_handler(code: int):
+    def error_handler(err):
+        return jsonify({"msg": err.description, "status": code}), code
+
+    return error_handler
+
+
+def register_all_error_handlers(app):
+    """
+    function to register all handlers
+    """
+
+    error_codes_to_override = [404, 403, 401, 405, 400, 409, 422]
+
+    for code in error_codes_to_override:
+        app.register_error_handler(code, get_standard_error_handler(code))

svm_prefilter_service/utils/healthchecks.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+from healthcheck import HealthCheck
+from flask import Flask
+
+
+def dummy_health_check():
+    return True, "very good"
+
+
+def register_health_checks(app: Flask):
+    health = HealthCheck()
+    health.add_check(dummy_health_check)
+    app.add_url_rule("/healthz", "healthcheck", view_func=lambda: health.run())

svm_prefilter_service/utils/require_decorators.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+from flask import request, abort
+
+from functools import wraps
+
+
+def json_required(f):
+    @wraps(f)
+    def call(*args, **kwargs):
+
+        if request.is_json:
+            return f(*args, **kwargs)
+        else:
+            abort(400, "JSON required")
+
+    return call

svm_prefilter_service/views/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+from .filter_view import FilterView

svm_prefilter_service/views/filter_view.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+from flask import jsonify, request, abort, current_app, Response
+from flask_classful import FlaskView
+from utils import json_required
+
+
+class FilterView(FlaskView):
+
+    @json_required
+    def post(self):
+        data = request.json
+
+        return Response(status=201)