Merge branch 'master' of kmlabz:GoldenPogacsa/consumer-api

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit=venv/*

.drone.yml

@@ -3,32 +3,103 @@ type: docker
 name: default
 
 steps:
+- name: restore-cache-with-filesystem
+  image: meltwater/drone-cache
+  settings:
+    backend: "filesystem"
+    restore: true
+    cache_key: "{{ .Repo.Name }}"
+    archive_format: "gzip"
+    filesystem_cache_root: "/tmp/cache"
+    mount:
+      - '.pipcache'
+  volumes:
+  - name: cache
+    path: /tmp/cache
+
+
 - name: static_analysis
   image: python:3
   commands:
-  - pip3 install pylint bandit mccabe
-  - pip3 install -r requirements.txt
+  - pip3 install --cache-dir='./.pipcache' pylint bandit mccabe
+  - pip3 install --cache-dir='./.pipcache' -r requirements.txt
   - find . -name "*.py" -exec python3 -m py_compile '{}' \;
   - find . -name "*.py" -exec pylint '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi
   - find . -name "*.py" -exec python3 -m mccabe --min 3 '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi
   - bandit -r . + || if [ $? -eq 1 ]; then echo "you fail"; fi
 
-- name: build
+- name: unit_test
+  image: python:3.8
+  environment:
+    PRODUCER_REDIS: cache
+  commands:
+  - pip3 install --cache-dir='./.pipcache' -r requirements.txt
+  - pip3 install --cache-dir='./.pipcache' -r requirements_dev.txt
+  - pytest
+
+- name: coverage
+  image: python:3.8
+  environment:
+    PRODUCER_REDIS: cache
+  commands:
+    - pip3 install --cache-dir='./.pipcache' -r requirements.txt
+    - pip3 install --cache-dir='./.pipcache' -r requirements_dev.txt
+    - coverage run -m pytest
+    - coverage report -m
+
+- name: build-app
+  image: banzaicloud/drone-kaniko
+  settings:
+    registry: registry.kmlabz.com
+    repo: goldenpogacsa/${DRONE_REPO_NAME}
+    username:
+      from_secret: DOCKER_USERNAME
+    password:
+      from_secret: DOCKER_PASSWORD
+    tags:
+      - latest
+      - ${DRONE_BUILD_NUMBER}
+
+- name: make_docs
+  image: python:3.8
+  commands:
+  - pip3 install --cache-dir='./.pipcache' Sphinx sphinx_rtd_theme
+  - pip3 install -r requirements.txt
+  - cd docs
+  - make html
+
+- name: rebuild-cache-with-filesystem
+  image: meltwater/drone-cache
+  pull: true
+  settings:
+    backend: "filesystem"
+    rebuild: true
+    cache_key: "{{ .Repo.Name }}"
+    archive_format: "gzip"
+    filesystem_cache_root: "/tmp/cache"
+    mount:
+      - '.pipcache'
+  volumes:
+  - name: cache
+    path: /tmp/cache
+
+- name: build_docs
   image: docker:stable-dind
   volumes:
-  - name: dockersock
-    path: /var/run
+    - name: dockersock
+      path: /var/run
   environment:
-      DOCKER_USERNAME:
-        from_secret: DOCKER_USERNAME
-      DOCKER_PASSWORD:
-        from_secret: DOCKER_PASSWORD
+    DOCKER_USERNAME:
+      from_secret: DOCKER_USERNAME
+    DOCKER_PASSWORD:
+      from_secret: DOCKER_PASSWORD
   commands:
-  - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-  - docker build -t="$DOCKER_USERNAME/consumer_api" .
-  - docker build -t="$DOCKER_USERNAME/consumer_api:$DRONE_BUILD_NUMBER" .
-  - docker push "$DOCKER_USERNAME/consumer_api"
-  - docker push "$DOCKER_USERNAME/consumer_api:$DRONE_BUILD_NUMBER"
+    - cd docs
+    - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin registry.kmlabz.com
+    - docker build -t="registry.kmlabz.com/goldenpogacsa/consumer-api-docs" .
+    - docker build -t="registry.kmlabz.com/goldenpogacsa/consumer-api-docs:$DRONE_BUILD_NUMBER" .
+    - docker push "registry.kmlabz.com/goldenpogacsa/consumer-api-docs"
+    - docker push "registry.kmlabz.com/goldenpogacsa/consumer-api-docs:$DRONE_BUILD_NUMBER"
 
 - name: slack
   image: plugins/slack
@@ -51,3 +122,6 @@ services:
 volumes:
 - name: dockersock
   temp: {}
+- name: cache
+  host:
+    path: "/tmp/cache"

README.rst

@@ -0,0 +1,13 @@
+================
+P2P Consumer API 
+================
+
+Repository for the consumer's API
+
+implemented interfaces / functionalities:
+
+* consumer
+* log
+* sync
+
+Produced by GoldenPogácsa Inc.

consumer_api/app.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
 from flask import Flask
 import os
 from db import redis_client
@@ -14,11 +15,22 @@ __copyright__ = "Copyright 2020, GoldenPogácsa Team"
 __module_name__ = "app"
 __version__text__ = "1"
 
-sentry_sdk.init("https://0a106e104e114bc9a3fa47f9cb0db2f4@sentry.kmlabz.com/10")
+# Setup sentry
+SENTRY_DSN = os.environ.get("SENTRY_DSN")
+if SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=SENTRY_DSN,
+        integrations=[FlaskIntegration()],
+        send_default_pii=True,
+        release=os.environ.get('RELEASE_ID', 'test'),
+        environment=os.environ.get('RELEASEMODE', 'dev')
+    )
 
 app = Flask(__name__)
 app.config['REDIS_URL'] = os.environ['REDIS_URL']
 app.config['LOCAL_UUID'] = os.environ['LOCAL_UUID']
+app.config['CUSTOMER_TIMEOUT'] = int(os.environ.get('CUSTOMER_TIMEOUT', 30))
+app.config['PRODUCER_TIMEOUT'] = int(os.environ.get('PRODUCER_TIMEOUT', 60))
 
 redis_client.init_app(app)
 

consumer_api/tests/__init__.py

@@ -0,0 +1,5 @@
+import os
+
+# Setup environment variables for testing
+os.environ["LOCAL_UUID"] = "d8b2e5e2-f675-4194-9324-af58e4b70c54"
+os.environ["REDIS_URL"] = "redis://192.168.111.121/0"

consumer_api/tests/consumers_view_test.py

@@ -0,0 +1,22 @@
+import db
+import pytest
+
+
+
+@pytest.fixture
+def client(mocker):
+    mocker.patch("db.redis_client")
+    db.redis_client.get.side_effect=lambda a: None
+
+    from app import app
+
+    app.config['TESTING'] = True
+
+    with app.test_client() as client:
+        yield client
+
+
+def test_response_length(client):
+    r = client.get('/consumers')
+
+    assert r.status_code == 200

consumer_api/tests/log_view_test.py

@@ -0,0 +1,30 @@
+import db
+import pytest
+
+@pytest.fixture
+def client(mocker):
+    mocker.patch("db.redis_client")
+    db.redis_client.get.side_effect=lambda a: None
+
+    from app import app
+
+    app.config['TESTING'] = True
+
+    with app.test_client() as client:
+        yield client
+
+
+def test_log_code_get(client):
+    r = client.get('/log')
+
+    assert r.status_code == 405
+
+
+def test_log_code_post(client):
+    data = {
+        "uuid": "asdasdasd",
+        "message": "Hello There!"
+    }
+    r = client.post('/log', json=data)
+
+    assert r.status_code == 204

consumer_api/tests/sync_view_test.py

@@ -0,0 +1,30 @@
+import db
+import pytest
+
+@pytest.fixture
+def client(mocker):
+    mocker.patch("db.redis_client")
+    db.redis_client.get.side_effect=lambda a: None
+
+    from app import app
+
+    app.config['TESTING'] = True
+
+    with app.test_client() as client:
+        yield client
+
+
+
+def test_log_code_get(client):
+    r = client.get('/sync')
+
+    assert r.status_code == 405
+
+
+def test_log_code_post(client):
+    data = {
+        "uuid": "tesuuid"
+    }
+    r = client.post('/sync', json=data)
+
+    assert r.status_code == 200

consumer_api/views/consumers_view.py

@@ -1,13 +1,19 @@
 import json
 
 from db import redis_client
-from flask import jsonify, current_app
+from flask import jsonify
 from flask_classful import FlaskView
 
 
 class ConsumersView(FlaskView):
 
     def get(self):
-        consumer_list = json.loads((redis_client.get("consumer_list") or b"{}").decode('utf-8'))
-        current_app.logger.warning(jsonify([v['ip'] for k, v in consumer_list.items()]))
-        return jsonify([v['ip'] for k, v in consumer_list.items()])
+        keys = redis_client.keys('consumer_*')
+
+        list_of_customer_ips = []
+
+        for key in keys:
+            info = json.loads((redis_client.get(key) or b"{}").decode('utf-8'))
+            list_of_customer_ips.append(info['ip'])
+
+        return jsonify(list_of_customer_ips)

consumer_api/views/log_view.py

@@ -19,13 +19,16 @@ class LogView(FlaskView):
         if not last_known_remote_ip:
             current_app.logger.info(f"New producer {remote_uuid} at {remote_ip}")
         elif last_known_remote_ip != remote_ip:
-            current_app.logger.info(f"IP address of producer {remote_uuid} have changed: {last_known_remote_ip} -> {remote_ip}")
+            current_app.logger.info(
+                f"IP address of producer {remote_uuid} have changed: {last_known_remote_ip} -> {remote_ip}")
 
         # update expirity
         redis_client.set(prod_key, remote_ip.encode('utf-8'))
-        redis_client.expire(prod_key, 240)
+        redis_client.expire(prod_key, current_app.config["PRODUCER_TIMEOUT"])
 
         # print out message
         current_app.logger.info(f"New message: {request.json['message']}")
 
-        return Response(status=204)
+        # return HTTP 204 - No content message
+        return Response(status = 204)
+

consumer_api/views/sync_view.py

@@ -1,32 +1,46 @@
 import json
+import time
 from flask import request, current_app, jsonify
 from flask_classful import FlaskView
-from db import redis_client  # ez nagyon otvar
+from db import redis_client
 
 
 class SyncView(FlaskView):
 
     def post(self):
         remote_uuid = request.json['uuid']
-        remote_ip = request.remote_addr
 
-        consumer_list = json.loads((redis_client.get("consumer_list") or b"{}").decode('utf-8'))
+        if 'ip' in request.json:
+            remote_ip = request.json['ip']
 
-        # Log something about it
-        if remote_uuid not in consumer_list.keys():
+            if request.remote_addr != remote_ip:
+                current_app.logger.debug(f"IP was overriden by the remote consumer {remote_ip} instead of {request.remote_addr}")
+
+        else:
+            remote_ip = request.remote_addr
+
+        cust_key = f"consumer_{remote_uuid}"
+
+        last_known_info = json.loads((redis_client.get(cust_key) or b"{}").decode('utf-8'))
+
+        if not last_known_info:
             current_app.logger.info(f"New consumer registered (unknown UUID): {remote_uuid} at {remote_ip}")
-        else:  # known
-            if consumer_list[remote_uuid]['ip'] != remote_ip:
+
+        else:
+            if last_known_info['ip'] != remote_ip:
                 current_app.logger.info(f"Address of consumer {remote_uuid} changed to {remote_ip}")
 
-        consumer_list.update(
-            {remote_uuid: {"ip": remote_ip}}
-        )
+        info = {
+            "uuid": remote_uuid,
+            "ip": remote_ip,
+            "last_seen": time.time()
+        }
 
-        redis_client.set("consumer_list", json.dumps(consumer_list).encode('utf-8'))
+        redis_client.set(cust_key, json.dumps(info).encode('utf-8'))
+        redis_client.expire(cust_key, current_app.config["CUSTOMER_TIMEOUT"])
 
+        # return with the current UUID
         response = {
             "uuid": current_app.config['LOCAL_UUID']
         }
-
         return jsonify(response)

docs/Dockerfile

@@ -0,0 +1,3 @@
+FROM nginx:latest
+
+COPY _build/html/ /usr/share/nginx/html

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,55 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'Consumer API'
+copyright = '2020, GoldenPogacsa Inc.'
+author = 'GoldenPogacsa Inc.'
+
+# The full version, including alpha/beta/rc tags
+release = '0.1.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = []
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []

docs/index.rst

@@ -0,0 +1,23 @@
+.. Consumer API documentation master file, created by
+   sphinx-quickstart on Wed Apr  8 21:31:18 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Consumer API's documentation!
+========================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   readme
+   source/modules
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+* `Git repository <https://git.kmlabz.com/GoldenPogacsa/consumer-api>`_

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/readme.rst

@@ -0,0 +1 @@
+.. include:: ../README.rst

docs/source/app.rst

@@ -0,0 +1,7 @@
+app module
+==========
+
+.. automodule:: app
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/db.rst

@@ -0,0 +1,7 @@
+db module
+=========
+
+.. automodule:: db
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/modules.rst

@@ -0,0 +1,9 @@
+consumer_api
+============
+
+.. toctree::
+   :maxdepth: 4
+
+   app
+   db
+   views

docs/source/views.rst

@@ -0,0 +1,38 @@
+views package
+=============
+
+Submodules
+----------
+
+views.consumers\_view module
+----------------------------
+
+.. automodule:: views.consumers_view
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+views.log\_view module
+----------------------
+
+.. automodule:: views.log_view
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+views.sync\_view module
+-----------------------
+
+.. automodule:: views.sync_view
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: views
+   :members:
+   :undoc-members:
+   :show-inheritance:

requirements_dev.txt

@@ -0,0 +1,4 @@
+pytest
+pytest-mock
+mock
+coverage