Initial commit

.dockerignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+#Pycharm
+.idea/
+
+*.md
+.gitignore
+.git/
+*.yml
+contrib/*
+postman/*

.drone.yml

@@ -0,0 +1,33 @@
+kind: pipeline
+type: docker
+name: default
+
+steps:
+- name: code-analysis
+  image: aosapps/drone-sonar-plugin
+  settings:
+    sonar_host:
+      from_secret: SONAR_HOST
+    sonar_token:
+      from_secret: SONAR_CODE
+
+- name: kaniko
+  image: banzaicloud/drone-kaniko
+  settings:
+    registry: registry.kmlabz.com
+    repo: birbnetes/${DRONE_REPO_NAME}
+    username:
+      from_secret: DOCKER_USERNAME
+    password:
+      from_secret: DOCKER_PASSWORD
+    tags:
+      - latest
+      - ${DRONE_BUILD_NUMBER}
+
+- name: ms-teams
+  image: kuperiu/drone-teams
+  settings:
+    webhook: 
+      from_secret: TEAMS_WEBHOOK
+  when:
+    status: [ failure ]

.gitignore

@@ -0,0 +1,143 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+#Pycharm
+.idea/
+*.iml
+
+# Wing project file
+*.wpr
+
+# Nano, vi, vim lockfile
+*.swp
+
+# log
+*.log
+

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.9-slim
+
+ADD birb_latency_collector requirements.txt /birb_latency_collector/
+WORKDIR /birb_latency_collector/
+
+ENV PIP_NO_CACHE_DIR=true
+ENV TZ Europe/Budapest
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN pip3 install -r requirements.txt
+
+ENV GUNICORN_LOGLEVEL="info"
+
+EXPOSE 8000
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "--log-level", "${GUNICORN_LOGLEVEL}", "app:app"]
+

birb_latency_collector/app.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+from flask import Flask
+from werkzeug.middleware.proxy_fix import ProxyFix
+
+# import stuff
+from utils import register_all_error_handlers, register_health_checks, influxdb_instance
+
+# import views
+from views import ReportView
+
+from config import Config
+
+# Tracing stuffs
+import jaeger_client
+import opentracing
+from flask_opentracing import FlaskTracing
+
+# Setup sentry
+if Config.SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=Config.SENTRY_DSN,
+        integrations=[FlaskIntegration()],
+        traces_sample_rate=0,
+        send_default_pii=True,
+        release=Config.RELEASE_ID,
+        environment=Config.RELEASEMODE
+    )
+
+# create flask app
+app = Flask(__name__)
+app.config.from_object(Config)
+app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
+
+# init stuff
+influxdb_instance.init_app(app)
+
+
+# register error handlers
+register_all_error_handlers(app)
+
+
+# Setup tracing
+def initialize_tracer():
+    app.logger.info("Initializing jaeger...")
+    jaeger_cfg = jaeger_client.Config(config={}, service_name='birb-latency-collector', validate=True)
+    tracer = jaeger_cfg.initialize_tracer()
+    return tracer
+
+
+tracing = FlaskTracing(initialize_tracer, True, app)
+
+# register views
+for view in [ReportView]:
+    view.register(app, trailing_slash=False)
+
+register_health_checks(app)
+
+# start debugging if needed
+if __name__ == "__main__":
+    app.run(debug=True)
+else:
+    import os
+
+    if "gunicorn" in os.environ.get("SERVER_SOFTWARE", ""):
+        import logging
+
+        gunicorn_logger = logging.getLogger('gunicorn.error')
+        app.logger.handlers = gunicorn_logger.handlers
+        app.logger.setLevel(gunicorn_logger.level)
+
+        jaeger_logger = logging.getLogger('jaeger_tracing')
+        jaeger_logger.handlers = gunicorn_logger.handlers
+        jaeger_logger.setLevel(gunicorn_logger.level)
+
+        app.logger.info("Gunicorn environment detected!")
+    else:
+        app.logger.info("Not gunicorn")

birb_latency_collector/config.py

@@ -0,0 +1,17 @@
+import os
+
+
+class Config:
+    SECRET_KEY = os.environ.get('SECRET_KEY', os.urandom(12))
+
+    SENTRY_DSN = os.environ.get("SENTRY_DSN")
+    RELEASE_ID = os.environ.get("RELEASE_ID", "test")
+    RELEASEMODE = os.environ.get("RELEASEMODE", "dev")
+
+    INFLUXDB_HOST = os.environ['INFLUXDB_HOST']
+    INFLUXDB_PORT = os.environ['INFLUXDB_PORT']
+    INFLUXDB_USER = os.environ['INFLUXDB_USER']
+    INFLUXDB_PASSWORD = os.environ['INFLUXDB_PASSWORD']
+    INFLUXDB_DATABASE = os.environ['INFLUXDB_DATABASE']
+
+    INFLUXDB_TIMEOUT = 30

birb_latency_collector/schemas/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

birb_latency_collector/tests/__init__.py

@@ -0,0 +1 @@
+import os

birb_latency_collector/utils/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from .require_decorators import json_required
+from .error_handlers import register_all_error_handlers
+from .healthchecks import register_health_checks
+from .influx import influxdb_instance

birb_latency_collector/utils/error_handlers.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+from flask import jsonify
+
+
+def get_standard_error_handler(code: int):
+    def error_handler(err):
+        return jsonify({"msg": err.description, "status": code}), code
+
+    return error_handler
+
+
+def register_all_error_handlers(app):
+    """
+    function to register all handlers
+    """
+
+    error_codes_to_override = [404, 403, 401, 405, 400, 409, 422]
+
+    for code in error_codes_to_override:
+        app.register_error_handler(code, get_standard_error_handler(code))

birb_latency_collector/utils/healthchecks.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from .influx import influxdb_instance
+from healthcheck import HealthCheck
+from flask import Flask
+
+
+def health_database_status():
+    try:
+        version = influxdb_instance.connection.ping()
+    except Exception as e:
+        return False, str(e)
+    else:
+        return True, f'influxdb ({version}) is ok'
+
+
+def register_health_checks(app: Flask):
+    health = HealthCheck()
+    health.add_check(health_database_status)
+    app.add_url_rule("/healthz", "healthcheck", view_func=lambda: health.run())

birb_latency_collector/utils/influx.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+from flask_influxdb import InfluxDB
+
+influxdb_instance = InfluxDB()

birb_latency_collector/utils/require_decorators.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+from flask import request, abort
+
+from functools import wraps
+
+
+def json_required(f):
+    @wraps(f)
+    def call(*args, **kwargs):
+
+        if request.is_json:
+            return f(*args, **kwargs)
+        else:
+            abort(400, "JSON required")
+
+    return call

birb_latency_collector/views/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+from .report_view import ReportView

birb_latency_collector/views/report_view.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+from flask import jsonify, request, abort, current_app, Response
+from flask_classful import FlaskView
+
+
+class ReportView(FlaskView):
+    pass

k8s/configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: birb-latency-collector
+  labels:
+    app: birb-latency-collector
+  namespace: birbnetes
+data:
+  SENTRY_DSN:
+  RELEASE_ID: kmlabz-k8s
+  RELEASEMODE: release
+

k8s/deployment.yaml

@@ -0,0 +1,51 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: birb-latency-collector
+  namespace: birbnetes
+  labels:
+    app: birb-latency-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: birb-latency-collector
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: birb-latency-collector
+    spec:
+      containers:
+      - image: registry.kmlabz.com/birbnetes/birb-latency-collector
+        imagePullPolicy: Always
+        name: birb-latency-collector
+        envFrom:
+          - configMapRef:
+              name: birb-latency-collector
+        ports:
+        - containerPort: 8000
+      - name: jaeger-agent
+        image: jaegertracing/jaeger-agent:latest
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 5775
+          name: zk-compact-trft
+          protocol: UDP
+        - containerPort: 5778
+          name: config-rest
+          protocol: TCP
+        - containerPort: 6831
+          name: jg-compact-trft
+          protocol: UDP
+        - containerPort: 6832
+          name: jg-binary-trft
+          protocol: UDP
+        - containerPort: 14271
+          name: admin-http
+          protocol: TCP
+        args:
+          - --reporter.grpc.host-port=dns:///woolsey.tormakristof.eu:14250
+      imagePullSecrets:
+      - name: regcred

k8s/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: birb-latency-collector
+  namespace: birbnetes
+  labels:
+    app: birb-latency-collector
+spec:
+  ports:
+  - name: birb-latency-collector
+    port: 80
+    targetPort: 8000
+    protocol: TCP
+  selector:
+    app: birb-latency-collector
+  type: ClusterIP

requirements-dev.txt

@@ -0,0 +1 @@
+pytest

requirements.txt

@@ -0,0 +1,16 @@
+werkzeug~=2.0.1
+requests
+blinker
+Flask~=2.0.1
+marshmallow~=3.14.1
+Flask-Classful
+gunicorn
+sentry_sdk
+flask_minio
+py-healthcheck
+
+Flask-InfluxDB~=0.3.3
+
+jaeger-client
+Flask-Opentracing
+opentracing~=2.4.0