Merge branch 'master' of kmlabz:GoldenPogacsa/consumer-api

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit=venv/*

.drone.yml

@@ -3,41 +3,86 @@ type: docker
 name: default
 
 steps:
+- name: restore-cache-with-filesystem
+  image: meltwater/drone-cache
+  settings:
+    backend: "filesystem"
+    restore: true
+    cache_key: "{{ .Repo.Name }}"
+    archive_format: "gzip"
+    filesystem_cache_root: "/tmp/cache"
+    mount:
+      - '.pipcache'
+  volumes:
+  - name: cache
+    path: /tmp/cache
+
+
 - name: static_analysis
   image: python:3
   commands:
-  - pip3 install pylint bandit mccabe
+  - pip3 install --cache-dir='./.pipcache' pylint bandit mccabe
-  - pip3 install -r requirements.txt
+  - pip3 install --cache-dir='./.pipcache' -r requirements.txt
   - find . -name "*.py" -exec python3 -m py_compile '{}' \;
   - find . -name "*.py" -exec pylint '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi
   - find . -name "*.py" -exec python3 -m mccabe --min 3 '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi
   - bandit -r . + || if [ $? -eq 1 ]; then echo "you fail"; fi
 
-- name: build
+- name: unit_test
-  image: docker:stable-dind
+  image: python:3.8
-  volumes:
-  - name: dockersock
-    path: /var/run
   environment:
-      DOCKER_USERNAME:
+    PRODUCER_REDIS: cache
-        from_secret: DOCKER_USERNAME
-      DOCKER_PASSWORD:
-        from_secret: DOCKER_PASSWORD
   commands:
-  - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+  - pip3 install --cache-dir='./.pipcache' -r requirements.txt
-  - docker build -t="$DOCKER_USERNAME/consumer_api" .
+  - pip3 install --cache-dir='./.pipcache' -r requirements_dev.txt
-  - docker build -t="$DOCKER_USERNAME/consumer_api:$DRONE_BUILD_NUMBER" .
+  - pytest
-  - docker push "$DOCKER_USERNAME/consumer_api"
+
-  - docker push "$DOCKER_USERNAME/consumer_api:$DRONE_BUILD_NUMBER"
+- name: coverage
+  image: python:3.8
+  environment:
+    PRODUCER_REDIS: cache
+  commands:
+    - pip3 install --cache-dir='./.pipcache' -r requirements.txt
+    - pip3 install --cache-dir='./.pipcache' -r requirements_dev.txt
+    - coverage run -m pytest
+    - coverage report -m
+
+- name: build-app
+  image: banzaicloud/drone-kaniko
+  settings:
+    registry: registry.kmlabz.com
+    repo: goldenpogacsa/${DRONE_REPO_NAME}
+    username:
+      from_secret: DOCKER_USERNAME
+    password:
+      from_secret: DOCKER_PASSWORD
+    tags:
+      - latest
+      - ${DRONE_BUILD_NUMBER}
 
 - name: make_docs
   image: python:3.8
   commands:
-  - pip3 install Sphinx sphinx_rtd_theme
+  - pip3 install --cache-dir='./.pipcache' Sphinx sphinx_rtd_theme
   - pip3 install -r requirements.txt
   - cd docs
   - make html
 
+- name: rebuild-cache-with-filesystem
+  image: meltwater/drone-cache
+  pull: true
+  settings:
+    backend: "filesystem"
+    rebuild: true
+    cache_key: "{{ .Repo.Name }}"
+    archive_format: "gzip"
+    filesystem_cache_root: "/tmp/cache"
+    mount:
+      - '.pipcache'
+  volumes:
+  - name: cache
+    path: /tmp/cache
+
 - name: build_docs
   image: docker:stable-dind
   volumes:
@@ -50,11 +95,11 @@ steps:
       from_secret: DOCKER_PASSWORD
   commands:
     - cd docs
-    - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+    - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin registry.kmlabz.com
-    - docker build -t="$DOCKER_USERNAME/consumer-api-docs" .
+    - docker build -t="registry.kmlabz.com/goldenpogacsa/consumer-api-docs" .
-    - docker build -t="$DOCKER_USERNAME/consumer-api-docs:$DRONE_BUILD_NUMBER" .
+    - docker build -t="registry.kmlabz.com/goldenpogacsa/consumer-api-docs:$DRONE_BUILD_NUMBER" .
-    - docker push "$DOCKER_USERNAME/consumer-api-docs"
+    - docker push "registry.kmlabz.com/goldenpogacsa/consumer-api-docs"
-    - docker push "$DOCKER_USERNAME/consumer-api-docs:$DRONE_BUILD_NUMBER"
+    - docker push "registry.kmlabz.com/goldenpogacsa/consumer-api-docs:$DRONE_BUILD_NUMBER"
 
 - name: slack
   image: plugins/slack
@@ -77,3 +122,6 @@ services:
 volumes:
 - name: dockersock
   temp: {}
+- name: cache
+  host:
+    path: "/tmp/cache"

README.rst

@@ -10,3 +10,4 @@ implemented interfaces / functionalities:
 * log
 * sync
 
+Produced by GoldenPogácsa Inc.

consumer_api/app.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
 from flask import Flask
 import os
 from db import redis_client
@@ -14,11 +15,22 @@ __copyright__ = "Copyright 2020, GoldenPogácsa Team"
 __module_name__ = "app"
 __version__text__ = "1"
 
-sentry_sdk.init("https://0a106e104e114bc9a3fa47f9cb0db2f4@sentry.kmlabz.com/10")
+# Setup sentry
+SENTRY_DSN = os.environ.get("SENTRY_DSN")
+if SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=SENTRY_DSN,
+        integrations=[FlaskIntegration()],
+        send_default_pii=True,
+        release=os.environ.get('RELEASE_ID', 'test'),
+        environment=os.environ.get('RELEASEMODE', 'dev')
+    )
 
 app = Flask(__name__)
 app.config['REDIS_URL'] = os.environ['REDIS_URL']
 app.config['LOCAL_UUID'] = os.environ['LOCAL_UUID']
+app.config['CUSTOMER_TIMEOUT'] = int(os.environ.get('CUSTOMER_TIMEOUT', 30))
+app.config['PRODUCER_TIMEOUT'] = int(os.environ.get('PRODUCER_TIMEOUT', 60))
 
 redis_client.init_app(app)
 

consumer_api/tests/__init__.py

@@ -0,0 +1,5 @@
+import os
+
+# Setup environment variables for testing
+os.environ["LOCAL_UUID"] = "d8b2e5e2-f675-4194-9324-af58e4b70c54"
+os.environ["REDIS_URL"] = "redis://192.168.111.121/0"

consumer_api/tests/consumers_view_test.py

@@ -0,0 +1,22 @@
+import db
+import pytest
+
+
+
+@pytest.fixture
+def client(mocker):
+    mocker.patch("db.redis_client")
+    db.redis_client.get.side_effect=lambda a: None
+
+    from app import app
+
+    app.config['TESTING'] = True
+
+    with app.test_client() as client:
+        yield client
+
+
+def test_response_length(client):
+    r = client.get('/consumers')
+
+    assert r.status_code == 200

consumer_api/tests/log_view_test.py

@@ -0,0 +1,30 @@
+import db
+import pytest
+
+@pytest.fixture
+def client(mocker):
+    mocker.patch("db.redis_client")
+    db.redis_client.get.side_effect=lambda a: None
+
+    from app import app
+
+    app.config['TESTING'] = True
+
+    with app.test_client() as client:
+        yield client
+
+
+def test_log_code_get(client):
+    r = client.get('/log')
+
+    assert r.status_code == 405
+
+
+def test_log_code_post(client):
+    data = {
+        "uuid": "asdasdasd",
+        "message": "Hello There!"
+    }
+    r = client.post('/log', json=data)
+
+    assert r.status_code == 204

consumer_api/tests/sync_view_test.py

@@ -0,0 +1,30 @@
+import db
+import pytest
+
+@pytest.fixture
+def client(mocker):
+    mocker.patch("db.redis_client")
+    db.redis_client.get.side_effect=lambda a: None
+
+    from app import app
+
+    app.config['TESTING'] = True
+
+    with app.test_client() as client:
+        yield client
+
+
+
+def test_log_code_get(client):
+    r = client.get('/sync')
+
+    assert r.status_code == 405
+
+
+def test_log_code_post(client):
+    data = {
+        "uuid": "tesuuid"
+    }
+    r = client.post('/sync', json=data)
+
+    assert r.status_code == 200

consumer_api/views/consumers_view.py

@@ -1,15 +1,19 @@
 import json
 
 from db import redis_client
-from flask import jsonify, current_app
+from flask import jsonify
 from flask_classful import FlaskView
 
 
 class ConsumersView(FlaskView):
 
     def get(self):
-        # load the currently available consumer list from the redis database
+        keys = redis_client.keys('consumer_*')
-        consumer_list = json.loads((redis_client.get("consumer_list") or b"{}").decode('utf-8'))
 
-        # jsonify and return the list of active consumers
+        list_of_customer_ips = []
-        return jsonify([v['ip'] for k, v in consumer_list.items()])
+
+        for key in keys:
+            info = json.loads((redis_client.get(key) or b"{}").decode('utf-8'))
+            list_of_customer_ips.append(info['ip'])
+
+        return jsonify(list_of_customer_ips)

consumer_api/views/log_view.py

@@ -1,12 +1,34 @@
 from flask import request, current_app, Response
 from flask_classful import FlaskView
+from db import redis_client
 
 
 class LogView(FlaskView):
 
     def post(self):
-        # display received message
+        # Record the IP address of the producer
+        remote_uuid = request.json['uuid']
+        remote_ip = request.remote_addr
+
+        prod_key = f"producer_{remote_uuid}"
+
+        last_known_remote_ip = redis_client.get(prod_key)
+        if last_known_remote_ip:
+            last_known_remote_ip = last_known_remote_ip.decode('utf-8')
+
+        if not last_known_remote_ip:
+            current_app.logger.info(f"New producer {remote_uuid} at {remote_ip}")
+        elif last_known_remote_ip != remote_ip:
+            current_app.logger.info(
+                f"IP address of producer {remote_uuid} have changed: {last_known_remote_ip} -> {remote_ip}")
+
+        # update expirity
+        redis_client.set(prod_key, remote_ip.encode('utf-8'))
+        redis_client.expire(prod_key, current_app.config["PRODUCER_TIMEOUT"])
+
+        # print out message
         current_app.logger.info(f"New message: {request.json['message']}")
 
         # return HTTP 204 - No content message
         return Response(status = 204)
+

consumer_api/views/sync_view.py

@@ -1,4 +1,5 @@
 import json
+import time
 from flask import request, current_app, jsonify
 from flask_classful import FlaskView
 from db import redis_client
@@ -8,24 +9,35 @@ class SyncView(FlaskView):
 
     def post(self):
         remote_uuid = request.json['uuid']
-        remote_ip = request.remote_addr
 
-        # load the currently available consumer list from the redis database
+        if 'ip' in request.json:
-        consumer_list = json.loads((redis_client.get("consumer_list") or b"{}").decode('utf-8'))
+            remote_ip = request.json['ip']
+
+            if request.remote_addr != remote_ip:
+                current_app.logger.debug(f"IP was overriden by the remote consumer {remote_ip} instead of {request.remote_addr}")
 
-        if remote_uuid not in consumer_list.keys():
-            # display newly registered consumer
-            current_app.logger.info(f"New consumer registered (unknown UUID): {remote_uuid} at {remote_ip}")
         else:
-            if consumer_list[remote_uuid]['ip'] != remote_ip:
+            remote_ip = request.remote_addr
-                # log address changes
+
+        cust_key = f"consumer_{remote_uuid}"
+
+        last_known_info = json.loads((redis_client.get(cust_key) or b"{}").decode('utf-8'))
+
+        if not last_known_info:
+            current_app.logger.info(f"New consumer registered (unknown UUID): {remote_uuid} at {remote_ip}")
+
+        else:
+            if last_known_info['ip'] != remote_ip:
                 current_app.logger.info(f"Address of consumer {remote_uuid} changed to {remote_ip}")
 
-        # update consumer list redis databasse
+        info = {
-        consumer_list.update(
+            "uuid": remote_uuid,
-            {remote_uuid: {"ip": remote_ip}}
+            "ip": remote_ip,
-        )
+            "last_seen": time.time()
-        redis_client.set("consumer_list", json.dumps(consumer_list).encode('utf-8'))
+        }
+
+        redis_client.set(cust_key, json.dumps(info).encode('utf-8'))
+        redis_client.expire(cust_key, current_app.config["CUSTOMER_TIMEOUT"])
 
         # return with the current UUID
         response = {

requirements_dev.txt

@@ -0,0 +1,4 @@
+pytest
+pytest-mock
+mock
+coverage