initial commit

This commit is contained in:
Pünkösd Marcell 2021-11-23 00:19:29 +01:00
commit 851f451354
19 changed files with 605 additions and 0 deletions

139
.dockerignore Normal file
View File

@ -0,0 +1,139 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
#Pycharm
.idea/
*.md
.gitignore
.git/
*.yml
contrib/*
postman/*

33
.drone.yml Normal file
View File

@ -0,0 +1,33 @@
kind: pipeline
type: docker
name: default
steps:
- name: code-analysis
image: aosapps/drone-sonar-plugin
settings:
sonar_host:
from_secret: SONAR_HOST
sonar_token:
from_secret: SONAR_CODE
- name: kaniko
image: banzaicloud/drone-kaniko
settings:
registry: registry.kmlabz.com
repo: birbnetes/${DRONE_REPO_NAME}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
tags:
- latest
- ${DRONE_BUILD_NUMBER}
- name: ms-teams
image: kuperiu/drone-teams
settings:
webhook:
from_secret: TEAMS_WEBHOOK
when:
status: [ failure ]

143
.gitignore vendored Normal file
View File

@ -0,0 +1,143 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
#Pycharm
.idea/
*.iml
# Wing project file
*.wpr
# Nano, vi, vim lockfile
*.swp
# log
*.log

16
Dockerfile Normal file
View File

@ -0,0 +1,16 @@
FROM python:3.9
ADD svm_prefilter_service requirements.txt /svm_prefilter_service/
WORKDIR /svm_prefilter_service/
ENV PIP_NO_CACHE_DIR=true
ENV TZ Europe/Budapest
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN pip3 install -r requirements.txt
ENV GUNICORN_LOGLEVEL="info"
EXPOSE 8000
CMD ["gunicorn", "-b", "0.0.0.0:8000", "--log-level", "${GUNICORN_LOGLEVEL}", "app:app"]

11
k8s/configmap.yaml Normal file
View File

@ -0,0 +1,11 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: svm-prefilter-service
labels:
app: svm-prefilter-service
namespace: birbnetes
data:
SENTRY_DSN: ""
RELEASE_ID: kmlabz-k8s
RELEASEMODE: release

51
k8s/deployment.yaml Normal file
View File

@ -0,0 +1,51 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: svm-prefilter-service
namespace: birbnetes
labels:
app: svm-prefilter-service
spec:
replicas: 1
selector:
matchLabels:
app: svm-prefilter-service
strategy:
type: Recreate
template:
metadata:
labels:
app: svm-prefilter-service
spec:
containers:
- image: registry.kmlabz.com/birbnetes/svm_prefilter_service
imagePullPolicy: Always
name: svm-prefilter-service
envFrom:
- configMapRef:
name: svm-prefilter-service
ports:
- containerPort: 8000
- name: jaeger-agent
image: jaegertracing/jaeger-agent:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5775
name: zk-compact-trft
protocol: UDP
- containerPort: 5778
name: config-rest
protocol: TCP
- containerPort: 6831
name: jg-compact-trft
protocol: UDP
- containerPort: 6832
name: jg-binary-trft
protocol: UDP
- containerPort: 14271
name: admin-http
protocol: TCP
args:
- --reporter.grpc.host-port=dns:///woolsey.tormakristof.eu:14250
imagePullSecrets:
- name: regcred

16
k8s/service.yaml Normal file
View File

@ -0,0 +1,16 @@
apiVersion: v1
kind: Service
metadata:
name: svm-prefilter-service
namespace: birbnetes
labels:
app: svm-prefilter-service
spec:
ports:
- name: svm-prefilter-service
port: 80
targetPort: 8000
protocol: TCP
selector:
app: svm-prefilter-service
type: ClusterIP

1
requirements-dev.txt Normal file
View File

@ -0,0 +1 @@
pytest

38
requirements.txt Normal file
View File

@ -0,0 +1,38 @@
werkzeug~=2.0.1
requests
blinker
Flask~=2.0.1
marshmallow~=3.14.1
Flask-Classful
gunicorn
sentry_sdk
py-healthcheck
jaeger-client
Flask-Opentracing
opentracing~=2.4.0
cython
six
deprecation
cycler~=0.10.0
eyeD3==0.9.5
filetype==1.0.6
hmmlearn==0.2.3
joblib~=1.0.1
kiwisolver~=1.2.0
matplotlib~=3.3.3
numpy~=1.20.3
pydub==0.23.1
pyparsing==2.4.6
python-dateutil==2.8.1
scikit-learn~=0.24.0
scipy~=1.6.2
simplejson~=3.17.2
pyAudioAnalysis~=0.3.0
tqdm~=4.61.1

View File

@ -0,0 +1,78 @@
#!/usr/bin/env python3
import sentry_sdk
from sentry_sdk.integrations.flask import FlaskIntegration
from flask import Flask
from werkzeug.middleware.proxy_fix import ProxyFix
# import stuff
from utils import register_all_error_handlers, register_health_checks
# import views
from views import FilterView
from config import Config
# Tracing stuffs
import jaeger_client
import opentracing
from flask_opentracing import FlaskTracing
# Setup sentry
if Config.SENTRY_DSN:
sentry_sdk.init(
dsn=Config.SENTRY_DSN,
integrations=[FlaskIntegration()],
traces_sample_rate=0,
send_default_pii=True,
release=Config.RELEASE_ID,
environment=Config.RELEASEMODE
)
# create flask app
app = Flask(__name__)
app.config.from_object(Config)
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
# init stuff
# register error handlers
register_all_error_handlers(app)
# Setup tracing
def initialize_tracer():
app.logger.info("Initializing jaeger...")
jaeger_cfg = jaeger_client.Config(config={}, service_name='svm-prefilter-service', validate=True)
tracer = jaeger_cfg.initialize_tracer()
return tracer
tracing = FlaskTracing(initialize_tracer, True, app)
# register views
for view in [FilterView]:
view.register(app, trailing_slash=False)
register_health_checks(app)
# start debugging if needed
if __name__ == "__main__":
app.run(debug=True)
else:
import os
if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
import logging
gunicorn_logger = logging.getLogger('gunicorn.error')
app.logger.handlers = gunicorn_logger.handlers
app.logger.setLevel(gunicorn_logger.level)
jaeger_logger = logging.getLogger('jaeger_tracing')
jaeger_logger.handlers = gunicorn_logger.handlers
jaeger_logger.setLevel(gunicorn_logger.level)
app.logger.info("Gunicorn environment detected!")
else:
app.logger.info("Not gunicorn")

View File

@ -0,0 +1,9 @@
import os
class Config:
SECRET_KEY = os.environ.get('SECRET_KEY', os.urandom(12))
SENTRY_DSN = os.environ.get("SENTRY_DSN")
RELEASE_ID = os.environ.get("RELEASE_ID", "test")
RELEASEMODE = os.environ.get("RELEASEMODE", "dev")

View File

@ -0,0 +1 @@
#!/usr/bin/env python3

View File

@ -0,0 +1 @@
import os

View File

@ -0,0 +1,4 @@
#!/usr/bin/env python3
from .require_decorators import json_required
from .error_handlers import register_all_error_handlers
from .healthchecks import register_health_checks

View File

@ -0,0 +1,20 @@
#!/usr/bin/env python3
from flask import jsonify
def get_standard_error_handler(code: int):
def error_handler(err):
return jsonify({"msg": err.description, "status": code}), code
return error_handler
def register_all_error_handlers(app):
"""
function to register all handlers
"""
error_codes_to_override = [404, 403, 401, 405, 400, 409, 422]
for code in error_codes_to_override:
app.register_error_handler(code, get_standard_error_handler(code))

View File

@ -0,0 +1,13 @@
#!/usr/bin/env python3
from healthcheck import HealthCheck
from flask import Flask
def dummy_health_check():
return True, "very good"
def register_health_checks(app: Flask):
health = HealthCheck()
health.add_check(dummy_health_check)
app.add_url_rule("/healthz", "healthcheck", view_func=lambda: health.run())

View File

@ -0,0 +1,16 @@
#!/usr/bin/env python3
from flask import request, abort
from functools import wraps
def json_required(f):
@wraps(f)
def call(*args, **kwargs):
if request.is_json:
return f(*args, **kwargs)
else:
abort(400, "JSON required")
return call

View File

@ -0,0 +1,2 @@
#!/usr/bin/env python3
from .filter_view import FilterView

View File

@ -0,0 +1,13 @@
#!/usr/bin/env python3
from flask import jsonify, request, abort, current_app, Response
from flask_classful import FlaskView
from utils import json_required
class FilterView(FlaskView):
@json_required
def post(self):
data = request.json
return Response(status=201)