upload to docker hub

.drone.yml

@@ -24,6 +24,18 @@ steps:
         - latest
         - ${DRONE_BUILD_NUMBER}
 
+  - name: dockerhub
+    image: plugins/docker
+    settings:
+      repo: birbnetes/${DRONE_REPO_NAME}
+      username:
+        from_secret: DOCKERHUB_USER
+      password:
+        from_secret: DOCKERHUB_PASSWORD
+      tags:
+        - latest
+        - ${DRONE_BUILD_NUMBER}
+
   - name: ms-teams
     image: kuperiu/drone-teams
     settings:

Dockerfile

@@ -16,4 +16,4 @@ COPY ./src .
 
 EXPOSE 8080
 
-ENTRYPOINT ["gunicorn", "-b", "0.0.0.0:8080", "--workers", "1", "--threads", "1", "app:app"]
+ENTRYPOINT ["gunicorn", "-b", "0.0.0.0:8080", "app:app"]

requirements.txt

@@ -2,5 +2,12 @@ sentry_sdk[flask]
 gunicorn
 Flask
 Flask-RESTful
+sqlalchemy
+flask_sqlalchemy
+marshmallow
+marshmallow-sqlalchemy
+flask-marshmallow
 paho-mqtt
 flask-mqtt
+psycopg2-binary
+py-healthcheck

src/app.py

@@ -5,12 +5,17 @@ from flask_restful import Api
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
+from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
+from healthcheck import HealthCheck
 
 from resources import *
 from config import *
+from db import db
+from marshm import ma
 from mqtt_flask_instance import mqtt
 
 from mqtt_methods import handle_status_message
+from healthchecks import health_database_status
 
 """
 Main Flask RESTful API
@@ -28,7 +33,7 @@ if SENTRY_DSN:
     )
     sentry_sdk.init(
         dsn=SENTRY_DSN,
-        integrations=[FlaskIntegration(), sentry_logging],
+        integrations=[FlaskIntegration(), sentry_logging, SqlalchemyIntegration()],
         traces_sample_rate=1.0,
         send_default_pii=True,
         release=RELEASE_ID,
@@ -43,20 +48,22 @@ app.config['MQTT_BROKER_PORT'] = MQTT_PORT
 app.config['MQTT_USERNAME'] = MQTT_USERNAME
 app.config['MQTT_PASSWORD'] = MQTT_PASSWORD
 app.config['MQTT_REFRESH_TIME'] = 1.0  # refresh time in seconds
+app.config['SQLALCHEMY_DATABASE_URI'] = \
+    f"postgresql://{POSTGRES_USERNAME}:{POSTGRES_PASSWORD}@{POSTGRES_HOSTNAME}:5432/{POSTGRES_DB}?sslmode=require"
 
 api = Api(app)
+db.init_app(app)
+ma.init_app(app)
 mqtt.init_app(app)
+health = HealthCheck()
 
-api.add_resource(DeviceOfflineResrouce, "/devices/<string:deviceid>/offline")
+with app.app_context():
-api.add_resource(DeviceOnlineResrouce, "/devices/<string:deviceid>/online")
+    db.create_all()
-api.add_resource(AlertResource, "/devices/<string:deviceid>/alert")
-api.add_resource(SensorOfflineResource, "/devices/<string:deviceid>/<string:sensorid>/offline")
-api.add_resource(SensorOnlineResource, "/devices/<string:deviceid>/<string:sensorid>/online")
 
 
 @mqtt.on_log()
 def handle_logging(client, userdata, level, buf):
-    app.logger.log(level, buf)
+    logger.log(level, buf)
 
 
 @mqtt.on_connect()
@@ -71,3 +78,17 @@ def handle_status_message_proxy(*args, **kwargs):
     """
     with app.app_context():
         handle_status_message(*args, **kwargs)
+
+
+api.add_resource(AllDevicesResource, "/devices")
+api.add_resource(AllDevicesOfflineResource, "/devices/offline")
+api.add_resource(AllDevicesOnlineResource, "/devices/online")
+api.add_resource(DeviceResource, "/devices/{deviceid}")
+api.add_resource(DeviceOfflineResrouce, "/devices/{deviceid}/offline")
+api.add_resource(DeviceOnlineResrouce, "/devices/{deviceid}/online")
+api.add_resource(SensorResource, "/devices/{deviceid}/{sensorid}")
+api.add_resource(SensorOfflineResource, "/devices/{deviceid}/{sensorid}/offline")
+api.add_resource(SensorOnlineResource, "/devices/{deviceid}/{sensorid}/online")
+
+health.add_check(health_database_status)
+app.add_url_rule("/healthz", "healthcheck", view_func=lambda: health.run())

src/config.py

@@ -20,6 +20,11 @@ SENTRY_DSN = os.environ.get("SENTRY_DSN")
 RELEASE_ID = os.environ.get("RELEASE_ID", "test")
 RELEASEMODE = os.environ.get("CNC_SERVICE_RELEASEMODE", "dev")
 
+POSTGRES_HOSTNAME = os.getenv("CNC_POSTGRES_HOSTNAME", "localhost")
+POSTGRES_USERNAME = os.getenv("CNC_POSTGRES_USERNAME", "cnc-service")
+POSTGRES_PASSWORD = os.getenv("CNC_POSTGRES_PASSWORD", "cnc-service")
+POSTGRES_DB = os.getenv("CNC_POSTGRES_DB", "cnc-service")
+
 MQTT_HOSTNAME = os.getenv("CNC_MQTT_HOSTNAME", "localhost")
 MQTT_PORT = os.getenv("CNC_MQTT_PORT", "1883")
 MQTT_USERNAME = os.getenv("CNC_MQTT_USERNAME", "cnc-service")

src/db.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+from flask_sqlalchemy import SQLAlchemy
+
+"""
+SQLAlchemy definition
+"""
+
+__author__ = '@tormakris'
+__copyright__ = "Copyright 2020, Birbnetes Team"
+__module_name__ = "db"
+__version__text__ = "1"
+
+db = SQLAlchemy()

src/healthchecks.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+from db import db
+
+"""
+Healthchek functions
+"""
+
+__author__ = "@tormakris"
+__copyright__ = "Copyright 2020, Birbnetes Team"
+__module_name__ = "healthchecks"
+__version__text__ = "1"
+
+
+def health_database_status():
+    is_database_working = True
+    output = 'database is ok'
+    try:
+        db.session.execute('SELECT 1')
+    except Exception as e:
+        output = str(e)
+        is_database_working = False
+    return is_database_working, output

src/marshm.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+from flask_marshmallow import Marshmallow
+
+"""
+Marshmallow definition
+"""
+
+__author__ = '@tormakris'
+__copyright__ = "Copyright 2020, Birbnetes Team"
+__module_name__ = "marshm"
+__version__text__ = "1"
+
+ma = Marshmallow()

src/models.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import enum
+from sqlalchemy.sql import func
+from db import db
+
+"""
+SQLAlchemy models
+"""
+
+__author__ = '@tormakris'
+__copyright__ = "Copyright 2020, Birbnetes Team"
+__module_name__ = "models"
+__version__text__ = "1"
+
+
+class DeviceStatusEnum(enum.Enum):
+    error = "error"
+    online = "online"
+    offline = "offline"
+
+
+class SensorStatusEnum(enum.Enum):
+    unknown = "unknown"
+    online = "online"
+    offline = "offline"
+
+
+class Device(db.Model):
+    __tablename__ = 'device'
+    id = db.Column(db.Integer, primary_key=True)
+    deviceid = db.Column(db.String, nullable=False)
+    status = db.Column(db.Enum(DeviceStatusEnum), nullable=False)
+    url = db.Column(db.String, nullable=False)
+    lastupdate = db.Column(db.TIMESTAMP, nullable=False, server_default=func.now(), onupdate=func.current_timestamp())
+    sensors = db.relationship("Sensor")
+
+
+class Sensor(db.Model):
+    __tablename__ = 'sensor'
+    id = db.Column(db.Integer, primary_key=True)
+    sensorid = db.Column(db.String, nullable=False)
+    status = db.Column(db.Enum(SensorStatusEnum), nullable=False)
+    device_id = db.Column(db.Integer, db.ForeignKey('device.id'))

src/mqtt_methods.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 from flask import current_app as app
 import config
+from db import db
+from schemas import DeviceSchema, SensorSchema
 
 """
 MQTT Stuff
@@ -11,6 +13,9 @@ __copyright__ = "Copyright 2020, Birbnetes Team"
 __module_name__ = "mqtt_methods"
 __version__text__ = "1"
 
+deviceschema = DeviceSchema(many=False)
+sensorschema = SensorSchema(many=False)
+
 
 def handle_status_message(client, userdata, message):
     data = dict(
@@ -21,10 +26,19 @@ def handle_status_message(client, userdata, message):
     try:
         ids = data['topic'].replace(f"{config.MQTT_STATUS_TOPIC}/", "").split("/")
         if len(ids) == 1:
-            app.logger.info(f"Recieved status message from {ids[0]} it was: {data['payload']}.")
+            data['payload']['deviceID'] = ids[0]
+            status_message = deviceschema.load(data['payload'], session=db.session, transient=True).data
+            app.logger.info(f"Recieved status message from {data['payload']['deviceID']}, persisting to db.")
+            db.session.merge(status_message)
         else:
             if len(ids) == 2:
-                app.logger.info(
+                data['payload']['deviceID'] = ids[0]
-                    f"Recieved status message from sensor {ids[1]} on device {ids[0]} it was: {data['payload']}.")
+                data['payload']['sensorID'] = ids[1]
+                status_message = sensorschema.load(data['payload'], session=db.session, transient=True).data
+                app.logger.info(f"Recieved status message from sensor {data['payload']['sensorID']}, persisting to db.")
+                db.session.merge(status_message)
     except Exception as e:
+        db.session.rollback()
         app.logger.exception(e)
+    else:
+        db.session.commit()

src/resources.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
-import json
-
 from flask_restful import Resource
+from db import db
 from mqtt_flask_instance import mqtt
 import config
+import models
+import schemas
 
 """
 Flask Restful endpoints
@@ -15,6 +16,61 @@ __module_name__ = "resources"
 __version__text__ = "1"
 
 
+class AllDevicesResource(Resource):
+    """
+    Query all known devices
+    """
+    alldeviceschema = schemas.DeviceSchema(many=True)
+
+    def get(self):
+        """
+        Get all stored items
+        :return:
+        """
+        alldevices = models.Device.query.all()
+        return self.alldeviceschema.dump(list(alldevices)), 200
+
+
+class AllDevicesOfflineResource(Resource):
+    """
+    Shut down all devices
+    """
+    def post(self):
+        """
+        Shut down every device
+        :return:
+        """
+        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/+", {"command": "offline"})
+
+
+class AllDevicesOnlineResource(Resource):
+    """
+    Bring every device online
+    """
+    def post(self):
+        """
+        Bring every device online
+        :return:
+        """
+        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/+", {"command": "online"})
+
+
+class DeviceResource(Resource):
+    """
+    Query and control a particular device
+    """
+    deviceschema = schemas.DeviceSchema(many=False)
+
+    def get(self, deviceid: str):
+        """
+        Query a device
+        :param deviceid: UUID of device
+        :return:
+        """
+        device = models.Device.query.filter_by(id=deviceid).first_or_404()
+        return self.deviceschema.dump(device), 200
+
+
 class DeviceOfflineResrouce(Resource):
     """
     Bring a device offline
@@ -22,10 +78,11 @@ class DeviceOfflineResrouce(Resource):
     def post(self, deviceid: str):
         """
         Shut down a device
-        :param deviceid: ID of device
+        :param deviceid: UUID of device
         :return:
         """
-        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{deviceid}", json.dumps({"command": "offline"}))
+        device = db.session.query(models.Device.id).filter(str(models.Device.id) == deviceid).first_or_404()[0]
+        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{device}", {"command": "offline"})
 
 
 class DeviceOnlineResrouce(Resource):
@@ -35,10 +92,28 @@ class DeviceOnlineResrouce(Resource):
     def post(self, deviceid: str):
         """
         Bring a device online
-        :param deviceid: ID of device
+        :param deviceid: UUID of device
         :return:
         """
-        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{deviceid}", json.dumps({"command": "online"}))
+        device = db.session.query(models.Device.id).filter(str(models.Device.id) == deviceid).first_or_404()[0]
+        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{device}", {"command": "online"})
+
+
+class SensorResource(Resource):
+    """
+    Query and control a particular sensor of a device
+    """
+    sensorschema = schemas.SensorSchema(many=False)
+
+    def get(self, deviceid: str, sensorid: str):
+        """
+        Query a sensor
+        :param deviceid: UUID of device
+        :param sensorid: UUID of sensor
+        :return:
+        """
+        sensor = models.Sensor.query.filter_by(device_id=deviceid, sensorid=sensorid).first_or_404()
+        return self.sensorschema.dump(sensor)
 
 
 class SensorOfflineResource(Resource):
@@ -48,11 +123,13 @@ class SensorOfflineResource(Resource):
     def post(self, deviceid: str, sensorid: str):
         """
         Shut down a sensor of a device
-        :param deviceid: ID of device
+        :param deviceid: UUID of device
-        :param sensorid: ID of sensor
+        :param sensorid: UUID of sensor
         :return:
         """
-        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{deviceid}/{sensorid}", json.dumps({"command": "offline"}))
+        sensor = db.session.query(models.Sensor.device_id, models.Sensor.id).filter(
+            str(models.Sensor.device_id) == deviceid and str(models.Sensor.id) == sensorid).first_or_404()
+        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{sensor[0]}/{sensor[1]}", {"command": "offline"})
 
 
 class SensorOnlineResource(Resource):
@@ -62,21 +139,10 @@ class SensorOnlineResource(Resource):
     def post(self, deviceid: str, sensorid: str):
         """
         Bring a sensor online
-        :param deviceid: ID of device
+        :param deviceid: UUID of device
-        :param sensorid: ID of sensor
+        :param sensorid: UUID of sensor
         :return:
         """
-        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{deviceid}/{sensorid}", json.dumps({"command": "online"}))
+        sensor = db.session.query(models.Sensor.device_id, models.Sensor.id).filter(
-
+            str(models.Sensor.device_id) == deviceid and str(models.Sensor.id) == sensorid).first_or_404()
-
+        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{sensor[0]}/{sensor[1]}", {"command": "online"})
-class AlertResource(Resource):
-    """
-    Force alert on device
-    """
-    def post(self, deviceid: str):
-        """
-        Shut down a sensor of a device
-        :param deviceid: ID of device
-        :return:
-        """
-        mqtt.publish(f"{config.MQTT_COMMAND_TOPIC}/{deviceid}", json.dumps({"command": "doAlert"}))

src/schemas.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import models
+from marshm import ma
+from marshmallow import fields
+
+"""
+Schemas of SQLAlchemy objects
+"""
+
+
+__author__ = "@tormakris"
+__copyright__ = "Copyright 2020, Birbnetes Team"
+__module_name__ = "schemas"
+__version__text__ = "1"
+
+
+class SensorSchema(ma.SQLAlchemyAutoSchema):
+    """
+    Sensor schema autogenerated
+    """
+    class Meta:
+        model = models.Sensor
+        include_fk = True
+        exclude = ('id',)
+
+
+class DeviceSchema(ma.SQLAlchemyAutoSchema):
+    """
+    Device schema autogenerated
+    """
+    sensors = fields.Nested(SensorSchema, many=True)
+
+    class Meta:
+        model = models.Device
+        include_relationships = True
+        exclude = ('lastupdate',)