From e17cff4d073be7ce5e9e0980007bfcc18545608b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Fri, 17 Jul 2020 16:31:49 +0200 Subject: [PATCH] create project structure --- .dockerignore | 140 ++++++++++++++++++++++++++++++++++++++++++++ .drone.yml | 77 ++++++++++++++++++++++++ .gitignore | 1 + Dockerfile | 19 ++++++ README.md | 5 +- docker-compose.yml | 35 +++++++++++ k8s/configmap.yaml | 21 +++++++ k8s/deployment.yaml | 29 +++++++++ k8s/service.yaml | 16 +++++ requirements.txt | 11 ++++ src/app.py | 59 +++++++++++++++++++ src/config.py | 27 +++++++++ src/db.py | 13 ++++ src/marshm.py | 14 +++++ 14 files changed, 466 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .drone.yml create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 k8s/configmap.yaml create mode 100644 k8s/deployment.yaml create mode 100644 k8s/service.yaml create mode 100644 requirements.txt create mode 100644 src/app.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/marshm.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..51f8a0c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,140 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +#Pycharm +.idea/ + +*.md +.gitignore +.git/ +*.yml +contrib/* +postman/* +*.wav \ No newline at end of file diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..4e11b5a --- /dev/null +++ b/.drone.yml @@ -0,0 +1,77 @@ +kind: pipeline +type: docker +name: default + +steps: + - name: restore-cache-with-filesystem + image: meltwater/drone-cache + settings: + backend: "filesystem" + restore: true + cache_key: "{{ .Repo.Name }}" + archive_format: "gzip" + filesystem_cache_root: "/tmp/cache" + mount: + - '.pipcache' + volumes: + - name: cache + path: /tmp/cache + + - name: static_analysis + image: "python:3.8" + commands: + - pip3 install --cache-dir='./.pipcache' pylint bandit mccabe + - pip3 install --cache-dir='./.pipcache' -r requirements.txt + - find . -name "*.py" -exec python3 -m py_compile '{}' \; + - find . -name "*.py" -exec pylint '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi + - find . -name "*.py" -exec python3 -m mccabe --min 3 '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi + - bandit -r . + || if [ $? -eq 1 ]; then echo "you fail"; fi + + - name: code-analysis + image: aosapps/drone-sonar-plugin + settings: + sonar_host: + from_secret: SONAR_HOST + sonar_token: + from_secret: SONAR_CODE + + - name: rebuild-cache-with-filesystem + image: meltwater/drone-cache:dev + pull: true + settings: + backend: "filesystem" + rebuild: true + cache_key: "{{ .Repo.Name }}" + archive_format: "gzip" + filesystem_cache_root: "/tmp/cache" + mount: + - '.pipcache' + volumes: + - name: cache + path: /tmp/cache + + - name: kaniko + image: banzaicloud/drone-kaniko + settings: + registry: registry.kmlabz.com + repo: birbnetes/${DRONE_REPO_NAME} + username: + from_secret: DOCKER_USERNAME + password: + from_secret: DOCKER_PASSWORD + tags: + - latest + - ${DRONE_BUILD_NUMBER} + + - name: ms-teams + image: kuperiu/drone-teams + settings: + webhook: + from_secret: TEAMS_WEBHOOK + when: + status: [ failure ] + +volumes: + - name: cache + host: + path: "/tmp/cache" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 13d1490..f345153 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,4 @@ dmypy.json # Pyre type checker .pyre/ +.idea/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b8f92b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.8-slim + +ENV TZ Europe/Budapest +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +WORKDIR /app + +ARG RELEASE_ID +ENV RELEASE_ID ${RELEASE_ID:-""} + +COPY requirements.txt ./ + +RUN pip install --no-cache-dir -r requirements.txt + +COPY ./src . + +EXPOSE 8080 + +ENTRYPOINT ["gunicorn", "-b", "0.0.0.0:8080", "--workers", "1", "--threads", "1", "app:app"] \ No newline at end of file diff --git a/README.md b/README.md index 488fb53..a922f9f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ # cnc-service -Command and Control Service \ No newline at end of file +Command and Control Service + +https://pypi.org/project/Flask-MQTT/ +Flask-MQTT is currently not suitable for the use with multiple worker instances. So if you use a WSGI server like gevent or gunicorn make sure you only have one worker instance. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f5f9282 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +networks: + cnc: + external: false + +services: + postgres: + image: postgres:12 + restart: always + environment: + - POSTGRES_USER=cnc-service + - POSTGRES_PASSWORD=cnc-service + - POSTGRES_DB=cnc-service + networks: + - cnc + volumes: + - ./postres-volume:/var/lib/postgresql/data + + activemq: + image: registry.kmlabz.com/birbnetes/activemq-artemis + restart: always + networks: + - cnc + volumes: + - ./artemis-volume:/var/lib/artemis-instance + + cnc-service: + image: registry.kmlabz.com/birbnetes/activemq-artemis + restart: always + depends_on: + - activemq + - postgres + networks: + - cnc + ports: + - "127.0.0.1:8080:8080" \ No newline at end of file diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml new file mode 100644 index 0000000..e7aa177 --- /dev/null +++ b/k8s/configmap.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cnc-service + labels: + app: cnc-service + namespace: birbnetes +data: + SENTRY_DSN: https://3515f6d0619b44a98f598edac480df02@sentry.kmlabz.com/22 + RELEASE_ID: birb-k8s + CNC_SERVICE_RELEASEMODE: release + CNC_RABBITMQ_HOSTNAME: birb-rabbitmq + CNC_RABBITMQ_EXCHANGE: "wave" + CNC_RABBITMQ_QUEUE: wave-ready + CNC_RABBITMQ_USERNAME: user + CNC_RABBITMQ_PASSWORD: 1wZVQnP5vy + CNC_POSTGRES_HOSTNAME: cnc-postgres + CNC_POSTGRES_USERNAME: cnc-service + CNC_POSTGRES_PASSWORD: cnc-service-supersecret + CNC_POSTGRES_DB: cnc-service + CNC_STORAGE_HOSTNAME: storage-service \ No newline at end of file diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml new file mode 100644 index 0000000..1033f2a --- /dev/null +++ b/k8s/deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cnc-service + namespace: birbnetes + labels: + app: cnc-service +spec: + replicas: 1 + selector: + matchLabels: + app: cnc-service + strategy: + type: Recreate + template: + metadata: + labels: + app: cnc-service + spec: + containers: + - name: input-service + image: registry.kmlabz.com/birbnetesgit/cnc-service + envFrom: + - configMapRef: + name: cnc-service + ports: + - containerPort: 8080 + imagePullSecrets: + - name: regcred \ No newline at end of file diff --git a/k8s/service.yaml b/k8s/service.yaml new file mode 100644 index 0000000..2d5040d --- /dev/null +++ b/k8s/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: cnc-service + namespace: birbnetes + labels: + app: cnc-service +spec: + ports: + - name: cnc-service + port: 80 + targetPort: 8080 + protocol: TCP + selector: + app: cnc-service + type: ClusterIP \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..343dc1b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +sentry_sdk[flask] +gunicorn +Flask +Flask-RESTful +sqlalchemy +flask_sqlalchemy +marshmallow +marshmallow-sqlalchemy +flask-marshmallow +paho-mqtt +flask-mqtt \ No newline at end of file diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..1d9d4a1 --- /dev/null +++ b/src/app.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import logging +from flask import Flask +from flask_restful import Api +import sentry_sdk +from sentry_sdk.integrations.flask import FlaskIntegration + +from config import * +from db import db +from marshm import ma +""" +Main Flask RESTful API +""" + +__author__ = "@tormakris" +__copyright__ = "Copyright 2020, Birbnetes Team" +__module_name__ = "app" +__version__text__ = "1" + +if SENTRY_DSN: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[FlaskIntegration()], + send_default_pii=True, + release=RELEASE_ID, + environment=RELEASEMODE + ) + + +app = Flask(__name__) +app.config['SQLALCHEMY_DATABASE_URI'] = f"postgresql://{POSTGRES_USERNAME}:{POSTGRES_PASSWORD}@{POSTGRES_HOSTNAME}:5432/{POSTGRES_DB}" + +api = Api(app) +db.init_app(app) +ma.init_app(app) + +with app.app_context(): + db.create_all() + +formatter = logging.Formatter( + fmt="%(asctime)s - %(levelname)s - %(module)s - %(message)s" +) + +handler = logging.StreamHandler() +handler.setFormatter(formatter) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logger.addHandler(handler) + +# api.add_resource(SampleResource, "/sample") +# api.add_resource(SampleParameterResource, '/sample/') + +if __name__ == "__main__": + app.run( + debug=bool(DEBUG), + host="0.0.0.0", + port=int(PORT), + ) \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..cba05c7 --- /dev/null +++ b/src/config.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import os + + +""" +Configuration variables +""" + + +__author__ = "@tormakris" +__copyright__ = "Copyright 2020, Birbnetes Team" +__module_name__ = "config" +__version__text__ = "1" + + +PORT = os.environ.get("CNC_SERVICE_PORT", 8080) +DEBUG = os.environ.get("CNC_SERVICE_DEBUG", True) + + +SENTRY_DSN = os.environ.get("SENTRY_DSN") +RELEASE_ID = os.environ.get("RELEASE_ID", "test") +RELEASEMODE = os.environ.get("CNC_SERVICE_RELEASEMODE", "dev") + +POSTGRES_HOSTNAME = os.getenv("CNC_POSTGRES_HOSTNAME", "localhost") +POSTGRES_USERNAME = os.getenv("CNC_POSTGRES_USERNAME", "cnc-service") +POSTGRES_PASSWORD = os.getenv("CNC_POSTGRES_PASSWORD", "cnc-service") +POSTGRES_DB = os.getenv("CNC_POSTGRES_DB", "cnc-service") diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..e56aa1f --- /dev/null +++ b/src/db.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +from flask_sqlalchemy import SQLAlchemy + +""" +SQLAlchemy definition +""" + +__author__ = '@tormakris' +__copyright__ = "Copyright 2020, Birbnetes Team" +__module_name__ = "db" +__version__text__ = "1" + +db = SQLAlchemy() diff --git a/src/marshm.py b/src/marshm.py new file mode 100644 index 0000000..c996091 --- /dev/null +++ b/src/marshm.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +from flask_marshmallow import Marshmallow + +""" +Marshmallow definition +""" + +__author__ = '@tormakris' +__copyright__ = "Copyright 2020, Birbnetes Team" +__module_name__ = "marshm" +__version__text__ = "1" + +ma = Marshmallow()