From 211be6cf96cf7fcb12aea78e954d6c9572cfa146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Sun, 13 Jun 2021 20:01:23 +0200 Subject: [PATCH 1/6] add influx capability --- requirements.txt | 5 ++++- src/app.py | 9 ++++++++- src/config.py | 6 ++++++ src/db.py | 2 +- src/influxus.py | 15 +++++++++++++++ src/resources.py | 18 +++++++++++++++++- 6 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 src/influxus.py diff --git a/requirements.txt b/requirements.txt index c798801..fc2acde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,7 @@ psycopg2-binary marshmallow marshmallow-sqlalchemy flask-marshmallow -py-healthcheck \ No newline at end of file +py-healthcheck +Flask-InfluxDB +tzdata +tzlocal \ No newline at end of file diff --git a/src/app.py b/src/app.py index 1419e65..2ad60c0 100644 --- a/src/app.py +++ b/src/app.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import logging from flask import Flask from flask_restful import Api import sentry_sdk @@ -10,6 +9,7 @@ from healthcheck import HealthCheck from config import * from db import db from marshm import ma +from influxus import influx_db from resources import SampleResource, SampleParameterResource from healthchecks import health_database_status @@ -42,10 +42,17 @@ app.config['FLASK_PIKA_PARAMS'] = {'host': RABBITMQ_HOST, 'password': RABBITMQ_PASSWORD, 'port': 5672, 'virtual_host': '/'} +app.config['INFLUXDB_HOST']=INFLUXDB_HOST +app.config['INFLUXDB_PORT']=INFLUXDB_PORT +app.config['INFLUXDB_USER']=INFLUXDB_USERNAME +app.config['INFLUXDB_PASSWORD']=INFLUXDB_PASSWORD +app.config['INFLUXDB_DATABASE']=INFLUXDB_DB + api = Api(app) health = HealthCheck() db.init_app(app) ma.init_app(app) +influx_db.init_app(app) with app.app_context(): db.create_all() diff --git a/src/config.py b/src/config.py index 4659e8c..18fd005 100644 --- a/src/config.py +++ b/src/config.py @@ -33,3 +33,9 @@ POSTGRES_PASSWORD = os.getenv("INPUT_POSTGRES_PASSWORD", "input-service") POSTGRES_DB = os.getenv("INPUT_POSTGRES_DB", "input-service") STORAGE_HOSTNAME = os.getenv("INPUT_STORAGE_HOSTNAME", "localhost:8042") + +INFLUXDB_HOST = os.getenv("INFLUX_HOST", "input-influx") +INFLUXDB_PORT = os.getenv("INFLUX_PORT", "8086") +INFLUXDB_USERNAME = os.getenv("INFLUX_USERNAME", "input-service") +INFLUXDB_PASSWORD = os.getenv("INFLUX_PASSWORD", "input-service-supersecret") +INFLUXDB_DB = os.getenv("INFLUX_DB", "input-service") diff --git a/src/db.py b/src/db.py index acfa6d2..98196e0 100644 --- a/src/db.py +++ b/src/db.py @@ -2,7 +2,7 @@ from flask_sqlalchemy import SQLAlchemy """ -Flask Restful endpoints +Database api """ __author__ = '@tormakris' diff --git a/src/influxus.py b/src/influxus.py new file mode 100644 index 0000000..6dc116b --- /dev/null +++ b/src/influxus.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +from flask_influxdb import InfluxDB + + +""" +Influx api +""" + +__author__ = '@tormakris' +__copyright__ = "Copyright 2020, Birbnetes Team" +__module_name__ = "influxus" +__version__text__ = "1" + +influx_db = InfluxDB() diff --git a/src/resources.py b/src/resources.py index 36fbe34..24d06ae 100644 --- a/src/resources.py +++ b/src/resources.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 import json +from datetime import datetime +import tzlocal from xeger import Xeger from flask_restful import Resource from flask import request, current_app import requests import pika from db import db +from influxus import influx_db from models import SampleMetadata from schemas import SampleSchema, SampleMetadataSchema from config import * @@ -106,7 +109,20 @@ class SampleResource(Resource): routing_key='feature', body=json.dumps({'tag': generated_tag}).encode('UTF-8')) connection.close() - + influx_db.write_points( + [ + { + 'time': datetime.now(tz=tzlocal.get_localzone()), + 'measurement': 'cloudinput', + 'tags': { + 'device': desc['device_id'] + }, + 'fields': { + 'bruh': 1.0 + } + } + ] + ) except Exception as e: current_app.logger.exception(e) db.session.rollback() From 8e0b252c927877eda46292f6035594ec83c73ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Sun, 13 Jun 2021 20:07:21 +0200 Subject: [PATCH 2/6] refine influx stuff --- src/app.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/app.py b/src/app.py index 2ad60c0..de74f39 100644 --- a/src/app.py +++ b/src/app.py @@ -42,11 +42,11 @@ app.config['FLASK_PIKA_PARAMS'] = {'host': RABBITMQ_HOST, 'password': RABBITMQ_PASSWORD, 'port': 5672, 'virtual_host': '/'} -app.config['INFLUXDB_HOST']=INFLUXDB_HOST -app.config['INFLUXDB_PORT']=INFLUXDB_PORT -app.config['INFLUXDB_USER']=INFLUXDB_USERNAME -app.config['INFLUXDB_PASSWORD']=INFLUXDB_PASSWORD -app.config['INFLUXDB_DATABASE']=INFLUXDB_DB +app.config['INFLUXDB_HOST'] = INFLUXDB_HOST +app.config['INFLUXDB_PORT'] = INFLUXDB_PORT +app.config['INFLUXDB_USER'] = INFLUXDB_USERNAME +app.config['INFLUXDB_PASSWORD'] = INFLUXDB_PASSWORD +app.config['INFLUXDB_DATABASE'] = INFLUXDB_DB api = Api(app) health = HealthCheck() @@ -55,6 +55,7 @@ ma.init_app(app) influx_db.init_app(app) with app.app_context(): + influx_db.database.create(INFLUXDB_DB) db.create_all() api.add_resource(SampleResource, "/sample") From 0c4b036b4794b2d563fb87bb9dbf20b942b3816f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Mon, 19 Jul 2021 16:00:27 +0200 Subject: [PATCH 3/6] do not create influxdb db --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index de74f39..fa89528 100644 --- a/src/app.py +++ b/src/app.py @@ -55,7 +55,7 @@ ma.init_app(app) influx_db.init_app(app) with app.app_context(): - influx_db.database.create(INFLUXDB_DB) + # influx_db.database.create(INFLUXDB_DB) db.create_all() api.add_resource(SampleResource, "/sample") From 95b557e8ba1f6f03ff36d3ab517ee1f7556c2725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Mon, 19 Jul 2021 16:03:19 +0200 Subject: [PATCH 4/6] remove sentry --- .drone.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7036ab1..d39867e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -23,13 +23,6 @@ steps: tags: - latest - ${DRONE_BUILD_NUMBER} -- name: sentry - image: tormachris/drone-sentry - settings: - sentry_project: ${DRONE_REPO_NAME} - sentry_domain: sentry.kmlabz.com - sentry_token: - from_secret: SENTRY_TOKEN - name: ms-teams image: kuperiu/drone-teams From 7e61ee8a6e698ddfb1f78b1d6d5a277e2be9cf03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Mon, 19 Jul 2021 16:21:17 +0200 Subject: [PATCH 5/6] update postgresql opts --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index fa89528..0ae3052 100644 --- a/src/app.py +++ b/src/app.py @@ -35,7 +35,7 @@ if SENTRY_DSN: app = Flask(__name__) app.config[ - 'SQLALCHEMY_DATABASE_URI'] = f"postgresql://{POSTGRES_USERNAME}:{POSTGRES_PASSWORD}@{POSTGRES_HOSTNAME}:5432/{POSTGRES_DB}?sslmode=require" + 'SQLALCHEMY_DATABASE_URI'] = f"postgresql://{POSTGRES_USERNAME}:{POSTGRES_PASSWORD}@{POSTGRES_HOSTNAME}:5432/{POSTGRES_DB}{POSTGRES_OPTS}" app.config['EXCHANGE_NAME'] = RABBITMQ_EXCHANGE app.config['FLASK_PIKA_PARAMS'] = {'host': RABBITMQ_HOST, 'username': RABBITMQ_USERNAME, From 25aac079d163cd6d1a87138102e740e501b3c7e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torma=20Krist=C3=B3f?= Date: Mon, 19 Jul 2021 16:21:53 +0200 Subject: [PATCH 6/6] add opts --- src/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.py b/src/config.py index 18fd005..088d7e9 100644 --- a/src/config.py +++ b/src/config.py @@ -31,6 +31,7 @@ POSTGRES_HOSTNAME = os.getenv("INPUT_POSTGRES_HOSTNAME", "localhost") POSTGRES_USERNAME = os.getenv("INPUT_POSTGRES_USERNAME", "input-service") POSTGRES_PASSWORD = os.getenv("INPUT_POSTGRES_PASSWORD", "input-service") POSTGRES_DB = os.getenv("INPUT_POSTGRES_DB", "input-service") +POSTGRES_OPTS = os.getenv("INPUT_POSTGRES_OPTS", "") STORAGE_HOSTNAME = os.getenv("INPUT_STORAGE_HOSTNAME", "localhost:8042")