Compare commits

..

101 Commits

Author SHA1 Message Date
489883b464 lowered giveup time
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-14 23:02:52 +02:00
f3e8750ff1 pytest is missing for some reason
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-14 20:50:22 +02:00
c50cd6b23a install requirements in coverage step
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-14 20:48:47 +02:00
32f104df7c coverage
Some checks failed
continuous-integration/drone/push Build is failing
2020-05-14 20:46:34 +02:00
957227b715 Merge branch 'master' of gitea:GoldenPogacsa/producer
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-14 20:09:49 +02:00
6bdcb40220 add consumerinformation 2020-05-14 20:09:41 +02:00
0ab07cb1a2 Update 'README.rst'
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-14 13:43:54 +02:00
20a1aa6dfa Merge remote-tracking branch 'origin/master'
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-13 12:34:46 +02:00
f782d167ef Updated app doc 2020-05-13 12:34:31 +02:00
d82cc3910b use cache at unit_test step
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-08 23:34:06 +02:00
ea7982e0ef fix ci cache
Some checks reported errors
continuous-integration/drone/push Build was killed
2020-05-08 22:22:44 +02:00
926ed41a0f Merge remote-tracking branch 'origin/master'
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-08 20:57:46 +02:00
f015eb2301 Created integration test 2020-05-08 20:57:23 +02:00
4610c7a42f Fixed exist test in consumerlocator 2020-05-08 20:32:32 +02:00
112a7fc8c2 Merge pull request 'dev-ci' (#8) from dev-ci into master
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-08 19:34:11 +02:00
3d4274db86 add cache volume
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2020-05-08 19:32:13 +02:00
02e3dc5267 add cache to ci
Some checks failed
continuous-integration/drone/push Build is failing
2020-05-08 19:25:42 +02:00
f898af4558 Init integration test
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-08 19:21:38 +02:00
2cb6ef3a6c Added integration test skeleton
All checks were successful
continuous-integration/drone/push Build is passing
2020-05-08 19:13:00 +02:00
c5a3fb2d04 fix unit tests
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-29 12:31:33 +02:00
22ebf01e20 fix some tests 2020-04-29 12:16:27 +02:00
58c4e296ea use custom dind to build doc container image
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-28 20:53:51 +02:00
d0cdb61916 dockerfile keyword is still needed
Some checks reported errors
continuous-integration/drone/push Build was killed
2020-04-28 20:30:40 +02:00
e8f206728f change context of document building
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-28 20:26:41 +02:00
be8873956b change directory before kaniko step
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-28 20:13:55 +02:00
790cfa3a91 use kaniko to build container images
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-28 19:37:04 +02:00
8a5f9efd08 Added logging about failover
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-22 03:59:34 +02:00
49d686a4ec Fixed stuff probably not being saved 2020-04-22 03:55:33 +02:00
62d4bc48e8 This never returns with None 2020-04-22 03:43:20 +02:00
00300f5b2f If something expected happens than it should not be logged as ERROR 2020-04-22 03:41:35 +02:00
ad44b4f134 Moved learning about new consumers to every cycle
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-22 03:16:28 +02:00
d3656b543f Added timeout for requests
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-22 01:48:29 +02:00
448acd4dd0 dont test
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-21 22:46:49 +02:00
844341fdd0 Fixed test_updateconsumer
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-21 20:30:56 +02:00
010e0d708f Fixed redisconnector's currentconsumer handling 2020-04-21 20:30:03 +02:00
de4c2ca6fb Merge remote-tracking branch 'origin/master'
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-21 19:53:12 +02:00
20e7f2fa05 Change datetime import back 2020-04-21 19:52:55 +02:00
d72f45b4c2 Update 'README.rst'
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-21 19:50:55 +02:00
d69e97ca3b Change datetime import
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-21 19:36:42 +02:00
ed6ed3baf0 Fixed datetime json export
Some checks reported errors
continuous-integration/drone/push Build encountered an error
2020-04-21 19:25:27 +02:00
7e9354be18 Added uuid to MessageSender in tests
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-17 19:25:13 +02:00
f0822580f1 Remove redis_db from test arguments
Some checks reported errors
continuous-integration/drone/push Build encountered an error
2020-04-17 19:17:55 +02:00
215dbdbbf3 Fix redis mock is tests
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-17 19:08:20 +02:00
e65d48b334 Mock redis is tests
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-17 18:57:46 +02:00
2a95eb392a Mock redis is tests 2020-04-17 18:57:32 +02:00
96a91c4154 Merge branch 'master' of gitea:GoldenPogacsa/producer
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-17 17:01:01 +02:00
03082b346d send uuid with message 2020-04-17 16:59:35 +02:00
46ae52a537 Merge remote-tracking branch 'origin/master'
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-17 16:47:48 +02:00
4a3fc3b1ba Added redis to consumerlocator 2020-04-17 16:47:27 +02:00
72cc952506 more docker layers
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-17 16:35:42 +02:00
eff5f4191c Merge pull request 'redis communicator' (#7) from redis into master
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-17 16:20:43 +02:00
650c9e074d redis communicator done
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2020-04-17 16:18:54 +02:00
e910d91f06 Merge pull request 'docs' (#6) from docs into master
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-08 23:05:31 +02:00
b7d69406d5 Updated drone stuff to generate docs
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
hopefully
2020-04-08 21:07:54 +02:00
1584c44113 fixed docs
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2020-04-08 21:02:41 +02:00
95905f5045 Added link to git repo in doc
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-08 19:51:14 +02:00
b31213c55c Updated __init__ comments
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-08 19:45:05 +02:00
43daae8cb4 Include __init__ functions in doc 2020-04-08 19:44:37 +02:00
dfc3b73673 Fixed comment format (now renders properly)
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-08 18:17:39 +02:00
ccf6bc0fff Modified sphinx theme 2020-04-08 18:16:29 +02:00
393188d549 Merge pull request 'auto generate documentation docker image via ci' (#5) from docs into master
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-08 16:03:27 +02:00
4f3acebb90 auto generate documentation
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2020-04-08 16:01:01 +02:00
eb5b5af3d3 Merge pull request 'Merge testing into master for the docs' (#4) from testing into master
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-07 18:46:52 +02:00
0d1c464569 Fix drone.yml for merge with testing
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-07 18:46:31 +02:00
d1d8fa1779 Merge remote-tracking branch 'origin/testing' into testing
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2020-04-07 18:02:47 +02:00
9e45b4751e Added sphinx files 2020-04-07 18:01:06 +02:00
35fd32a4c1 added sphinx commands
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-07 17:57:48 +02:00
e2a6c96025 adde sphinx commands
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-07 17:56:38 +02:00
c9ba2566f6 Merge remote-tracking branch 'origin/testing' into testing
All checks were successful
continuous-integration/drone/push Build is passing
# Conflicts:
#	communicator.py
2020-04-06 20:51:09 +02:00
9040cf898b Added/update docstring 2020-04-06 20:48:36 +02:00
64f7d9b83e test them consumers, boy
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-01 02:23:38 +02:00
1e8e3bd86c consumers :)
Some checks failed
continuous-integration/drone/push Build is failing
2020-04-01 02:20:39 +02:00
e0c4d68da5 log exception
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-01 02:15:01 +02:00
65db908fb6 add more logging
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-01 02:05:15 +02:00
de64ef0e41 add date to log output
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-01 01:58:49 +02:00
87217d121d generate uuid
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-01 01:57:27 +02:00
5a7e1590e9 added minimal excpetion handling to communicator.py
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-01 01:03:50 +02:00
f2c82419c8 added minimal excpetion handling to communicator.py
All checks were successful
continuous-integration/drone/push Build is passing
2020-04-01 01:02:33 +02:00
a784cad088 added learnconsumerlist to init phase of app.py
All checks were successful
continuous-integration/drone/push Build is passing
2020-03-31 22:58:59 +02:00
bfbbb3f089 Merge pull request 'Create unit tests' (#3) from testing into master
All checks were successful
continuous-integration/drone/push Build is passing
2020-03-31 21:49:57 +02:00
7d19ec0fa8 Fixed tests
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2020-03-31 14:26:29 +02:00
7a7643cfeb See if currentconsumer is null after init
Some checks reported errors
continuous-integration/drone/push Build encountered an error
continuous-integration/drone/pr Build is failing
2020-03-31 12:24:53 +02:00
d47d2e8d97 Added not None assert to updateconsumertest
Some checks failed
continuous-integration/drone/pr Build is failing
continuous-integration/drone/push Build is failing
2020-03-31 11:46:19 +02:00
4d1a0e1d57 Fix updateconsumer text
Some checks reported errors
continuous-integration/drone/pr Build encountered an error
continuous-integration/drone/push Build is failing
2020-03-30 22:12:21 +02:00
bbfa138849 tests now run
Some checks reported errors
continuous-integration/drone/push Build encountered an error
continuous-integration/drone/pr Build is failing
2020-03-30 21:36:26 +02:00
782d652c0d extract envvar to module-level variable
Some checks reported errors
continuous-integration/drone/pr Build was killed
continuous-integration/drone/push Build was killed
2020-03-30 19:52:52 +02:00
50dec05e6b tests done
Some checks reported errors
continuous-integration/drone/pr Build was killed
continuous-integration/drone/push Build was killed
2020-03-30 19:33:09 +02:00
39d32f39f4 add mock libraries
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2020-03-30 17:43:30 +02:00
40db3cd26e add pytest dependency
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2020-03-30 17:37:58 +02:00
ed01ead04d Merge pull request 'Wire all components together' (#2) from wire_together into master
All checks were successful
continuous-integration/drone/push Build is passing
\O/
2020-03-30 17:34:35 +02:00
77f01db969 solve circular dependency
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
2020-03-30 17:15:59 +02:00
94edf03cf8 introduce type hinting to consumerlocator
All checks were successful
continuous-integration/drone/push Build is passing
2020-03-30 17:10:48 +02:00
62f059b461 merge PR #1 Communicator API impementation branch: dev-communicator
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2020-03-29 20:04:53 +02:00
fe3537ff31 communicator requests api implementation done
All checks were successful
continuous-integration/drone/push Build is passing
2020-03-29 20:00:27 +02:00
2b44a5bbad Added environment variable and fixed stuff
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is failing
2020-03-29 19:18:25 +02:00
48da2c1bb5 Merge branch 'master' of https://git.kmlabz.com/GoldenPogacsa/producer 2020-03-29 18:29:07 +02:00
9783b8c20b Fixed a lot of typos in consumerlocator and added some mocks 2020-03-29 18:27:19 +02:00
06671e29b8 Merge branch 'master' of gitea:GoldenPogacsa/producer
All checks were successful
continuous-integration/drone/push Build is passing
2020-03-29 18:26:22 +02:00
ef47d7d6d3 mock real behaviour 2020-03-29 18:26:19 +02:00
39c34f2a67 Merge remote-tracking branch 'origin/master'
All checks were successful
continuous-integration/drone/push Build is passing
2020-03-29 18:00:12 +02:00
36272830d5 Learn available consumers after switching current consumer 2020-03-29 18:00:00 +02:00
27 changed files with 1099 additions and 174 deletions

2
.coveragerc Normal file
View File

@ -0,0 +1,2 @@
[run]
omit=venv/*

View File

@ -3,17 +3,93 @@ type: docker
name: default
steps:
- name: restore-cache-with-filesystem
image: meltwater/drone-cache
settings:
backend: "filesystem"
restore: true
cache_key: "{{ .Repo.Name }}"
archive_format: "gzip"
filesystem_cache_root: "/tmp/cache"
mount:
- '.pipcache'
volumes:
- name: cache
path: /tmp/cache
- name: static_analysis
image: python:3
image: python:3.8
commands:
- pip3 install pylint bandit mccabe
- pip3 install -r requirements.txt
- pip3 install --cache-dir='./.pipcache' pylint bandit mccabe
- pip3 install --cache-dir='./.pipcache' -r requirements.txt
- find . -name "*.py" -exec python3 -m py_compile '{}' \;
- find . -name "*.py" -exec pylint '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi
- find . -name "*.py" -exec python3 -m mccabe --min 3 '{}' + || if [ $? -eq 1 ]; then echo "you fail"; fi
- bandit -r . + || if [ $? -eq 1 ]; then echo "you fail"; fi
- name: build
- name: unit_test
image: python:3.8
environment:
PRODUCER_REDIS: cache
commands:
- pip3 install --cache-dir='./.pipcache' -r requirements.txt
- pytest test.py
- name: coverage
image: python:3.8
environment:
PRODUCER_REDIS: cache
commands:
- pip3 install --cache-dir='./.pipcache' -r requirements.txt
- pip3 install --cache-dir='./.pipcache' coverage pytest
- coverage run -m pytest test.py
- coverage report -m
- name: integration_test
image: python:3.8
environment:
PRODUCER_REDIS: cache
commands:
- pip3 install --cache-dir='./.pipcache' -r requirements.txt
- pytest integtest.py
- name: build-app
image: banzaicloud/drone-kaniko
settings:
registry: registry.kmlabz.com
repo: goldenpogacsa/${DRONE_REPO_NAME}
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
tags:
- latest
- ${DRONE_BUILD_NUMBER}
- name: make_docs
image: python:3.8
commands:
- pip3 install --cache-dir='./.pipcache' Sphinx sphinx_rtd_theme
- pip3 install --cache-dir='./.pipcache' -r requirements.txt
- cd docs
- make html
- name: rebuild-cache-with-filesystem
image: meltwater/drone-cache
pull: true
settings:
backend: "filesystem"
rebuild: true
cache_key: "{{ .Repo.Name }}"
archive_format: "gzip"
filesystem_cache_root: "/tmp/cache"
mount:
- '.pipcache'
volumes:
- name: cache
path: /tmp/cache
- name: build_docs
image: docker:stable-dind
volumes:
- name: dockersock
@ -24,11 +100,12 @@ steps:
DOCKER_PASSWORD:
from_secret: DOCKER_PASSWORD
commands:
- echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
- docker build -t="$DOCKER_USERNAME/producer" .
- docker build -t="$DOCKER_USERNAME/producer:$DRONE_BUILD_NUMBER" .
- docker push "$DOCKER_USERNAME/producer"
- docker push "$DOCKER_USERNAME/producer:$DRONE_BUILD_NUMBER"
- cd docs
- echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin registry.kmlabz.com
- docker build -t="registry.kmlabz.com/goldenpogacsa/producer-docs" .
- docker build -t="registry.kmlabz.com/goldenpogacsa/producer-docs:$DRONE_BUILD_NUMBER" .
- docker push "registry.kmlabz.com/goldenpogacsa/producer-docs"
- docker push "registry.kmlabz.com/goldenpogacsa/producer-docs:$DRONE_BUILD_NUMBER"
- name: slack
image: plugins/slack
@ -47,7 +124,12 @@ services:
volumes:
- name: dockersock
path: /var/run
- name: cache
image: redis
volumes:
- name: dockersock
temp: {}
- name: cache
host:
path: "/tmp/cache"

1
.gitignore vendored
View File

@ -131,3 +131,4 @@ dmypy.json
#Pycharm
.idea/
*.iml
.coverage

View File

@ -1,9 +1,11 @@
FROM python:3
FROM python:3.8
WORKDIR /app
COPY . ./
COPY requirements.txt ./
RUN pip3 install --no-cache-dir -r requirements.txt
COPY . ./
CMD ["python3", "app.py"]

12
README.rst Normal file
View File

@ -0,0 +1,12 @@
============
P2P Producer
============
This repository contains the Producer part of the project. The module manages the list of consumers and
sends data to the currently active one. If that becomes unavailable, it chooses an other one to the data to.
Implementation is done in python, the code is put into Docker images (as the consumers). To run the full project
clone the main repository and run
*docker-compose up.*
Produced by GoldenPogácsa Inc.

72
app.py
View File

@ -1,13 +1,20 @@
#!/usr/bin/env python
"""
Main entry point, this module builds the producer from the submodules.
"""
import os
import random
import uuid
import logging
import sentry_sdk
import communicator
import consumerlocator
import messagesender
"""
Main Flask RESTful API
"""
import time
from consumerinformation import ConsumerInformation
from communicator import Communicator
from consumerlocator import ConsumerLocator
from messagesender import MessageSender
from redisconnector import RedisConnector
__author__ = "@tormakris"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
@ -15,21 +22,40 @@ __module_name__ = "app"
__version__text__ = "1"
sentry_sdk.init("https://3fa5ae886ba1489092ad49a93cb419c1@sentry.kmlabz.com/9")
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
)
LOGGER = logging.getLogger(__name__)
KNOWNCONSUMER = os.getenv("PRODUCER_KNOWNCONSUMER", '10.69.42.1')
if __name__ == "__main__":
print("Producer: init")
comm = communicator.Communicator()
conslist = consumerlocator.ConsumerLocator()
conslist.initCommunicator(comm)
message = messagesender.MessageSender()
"""
This is the producers entry point, initializes all the components (:class:`communicator.Communicator`,
:class:`consumerlocator.ConsumerLocator` and :class:`messagesender.MessageSender`) and sends infinite random
messages. Basically this is a big loop, learning about consumers ( :func:`consumerlocator.ConsumerLocator.learnconsumerlist` ),
updating the current ( :func:`consumerlocator.ConsumerLocator.updateconsumer` ) one and sending the message
( :func:`messagesender.MessageSender.sendmessage` ).
If the current consumer is unavailable, the update will change to an available one. To not flood the network with
infinite data, we some random time.
"""
LOGGER.info("Producer started")
generateduuid = str(uuid.uuid4())
redisconnector = RedisConnector()
consumerinfomation = ConsumerInformation(redisconnector=redisconnector)
communicator = Communicator(currentconsumer=KNOWNCONSUMER, uuid=generateduuid,
consumerinformation=consumerinfomation)
LOGGER.debug(f"My uuid is {generateduuid}")
messagesender = MessageSender(communicator=communicator, uuid=generateduuid)
consumerlocator = ConsumerLocator(communicator=communicator,
redisconnector=redisconnector)
conslist.learnconsumerlist()
print("Producer: started sending")
for i in range(30):
print("Producer: send %i message", i)
msg = message.CreateMessage(20)
if not conslist.checkcurrentconsumer():
conslist.updateconsumer()
comm.sendmessage(msg)
print("Producer: finished")
while True:
consumerlocator.learnconsumerlist()
LOGGER.info(f"Updating consumer list of {generateduuid}")
consumerlocator.updateconsumer()
LOGGER.info("Sending message to consumer")
messagesender.sendmessage()
time.sleep(random.random())

View File

@ -1,48 +1,108 @@
#!/usr/bin/env python
import requests
from singleton import Singleton
"""
Communicator module
"""
import logging
import requests
import requests.exceptions
from consumerinformation import ConsumerInformation
__author__ = "@tormakris"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "messagesender"
__version__text__ = "1"
class Communicator(Singleton):
logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)
class Communicator:
"""
Class handling low level communication with consumers.
"""
def __init__(self, currentconsumer: str, uuid: str, consumerinformation: ConsumerInformation):
"""**Constructor:**
Initializes the object.
:param currentconsumer: the current consumer's IP address as a string
:param uuid: string typed UUID.
"""
self.currenctconsumer = currentconsumer
self.uuid = uuid
self.consumerinformation = consumerinformation
def sendmessage(self, message: str) -> None:
"""Send message to the current consumer. Logs the process.
:param message: the message of type string that will be sent.
:return: None
"""
Send message to consumer.
:param message:
:return: none
"""
pass
currentconsumer = self.currenctconsumer
LOGGER.info(f"Sending message to {currentconsumer}")
for consumer in self.consumerinformation.getconsumerlist():
try:
postresponse = requests.post(f'http://{consumer}/log', json={'uuid': self.uuid, 'message': message},
timeout=5)
LOGGER.debug(f"Message status code is:{postresponse.status_code}")
if postresponse.status_code < 300:
return None
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
LOGGER.warning(f"Could not send message to {consumer}")
def discoveravailableconsumers(self) -> list:
"""Get the list of available consumer from the current primary consumer. Logs the received list.
:return: list of consumers' IP addresses
"""
Get the list of available consumer from the current primary consumer.
:return:
"""
pass
try:
currentconsumer = self.currenctconsumer
response = requests.get(f'http://{currentconsumer}/consumers', timeout=5)
json = response.json()
LOGGER.info(f"List of currently available consumers: {json}")
return json
except Exception as e:
LOGGER.warning("Could not query available consumer list!")
# LOGGER.exception(e)
return []
def isconsumeravailable(self) -> bool:
"""Readiness probe current consumer. Logs the result.
:return: True if available, False otherwise
"""
Readiness probe primary consumer.
:return:
"""
pass
currentconsumer = self.currenctconsumer
try:
response = requests.get(f'http://{currentconsumer}/consumers', timeout=5)
isavailable = response.status_code == 200
except Exception as e:
# LOGGER.exception(e)
isavailable = False
LOGGER.info(f"Current consumer availability: {isavailable}")
return isavailable
def checkconsumer(self, consumer: str) -> bool:
"""Readiness probe of a particular consumer. Logs the result.
:param consumer: the consumer's IP address
:return: True if available, False otherwise
"""
Readiness probe of a prticular consumer.
:param consumer:
:return:
try:
response = requests.get(f'http://{consumer}/consumers', timeout=5)
isavailable = response.status_code == 200
except Exception as e:
# LOGGER.exception(e)
isavailable = False
LOGGER.info(f"Consumer {consumer} availability: {isavailable}")
return isavailable
def set_currentconsumer(self, currenctconsumer) -> None:
"""Set current consumer
:param currenctconsumer: the consumer's IP address
:return: None
"""
pass
self.currenctconsumer = currenctconsumer

37
consumerinformation.py Normal file
View File

@ -0,0 +1,37 @@
#!/usr/bin/env python
"""
Consumer locator module, that manages the list of consumers.
"""
from redisconnector import RedisConnector
__author__ = "@ricsik52"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "consumerlocator"
__version__text__ = "1"
class ConsumerInformation:
"""
Component responsible for providing high level information about consumers
"""
def __init__(self, redisconnector: RedisConnector):
"""**Constructor:**
Initializes the object.
:param redisconnector: the :class:'redisconnector.RedisConnector' instance that will be used for Redis connection.
"""
self.red = redisconnector
def getconsumerlist(self) -> list:
"""
Gets the list of currently available consumers.
:return:
"""
toreturn = []
for consumer in self.red.consumerlist:
if consumer['State']:
toreturn.append(consumer['Host'])
return toreturn

View File

@ -1,95 +1,137 @@
#!/usr/bin/env python
import datetime
import communicator
"""
Consumer locator module, that manages the list of consumers.
"""
"""
Consumer locator modul, that manages the list of consumers.
"""
import datetime
from communicator import Communicator
import os
from redisconnector import RedisConnector
import logging
__author__ = "@dscharnitzky"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "consumerlocator"
__version__text__ = "1"
KNOWNCONSUMER = os.getenv("PRODUCER_KNOWNCONSUMER", '10.69.42.1')
LOGGER = logging.getLogger(__name__)
class ConsumerLocator:
"""
Manages the list of consumers.
Component responsible for managing the list of consumers. Requires an instance of :class:`communicator.Communicator`
"""
def __init__(self):
"""
Initialize class.
"""
self.consumerList = [{"Host": "KnownHost", "State": True, "LastOk": datetime.datetime.now()}]
self.currentConsumer = self.consumerList[0]["Host"]
def __init__(self, communicator: Communicator, redisconnector: RedisConnector):
"""**Constructor:**
Initializes the object.
def initCommunicator(self, comm: communicator.Communicator):
self.communicator = comm
Gets the known consumer's IP address from the PRODUCER_KNOWNCONSUMER envar.
def learnconsumerlist(self):
""""
Learns the list of consumers.
:param communicator: the :class:'communicator.Communicator' instance that will be used for the low level communication.
"""
#TODO improve learning
recievedConsumerList = self.communicator.discoveravailableconsumers()
for consumer in recievedConsumerList:
self.consumerList.append({"Host": consumer, "State": True, "LastOk": datetime.datetime.now()})
self.updateConsumerList()
self.red = redisconnector
self.red.consumerlist = [{"Host": KNOWNCONSUMER, "State": True, "LastOk": datetime.datetime.now().timestamp()}]
self.red.currentconsumer = self.red.consumerlist[0]
self.communicator = communicator
def updateconsumerlist(self):
def learnconsumerlist(self) -> None:
""""Learns the list of consumers from the current consumer.
Calls :func:`communicator.Communicator.didiscoveravailableconsumers`, adds the learned consumers to the list
if are not present, and then calls :func:`consumerlocator.ConsumerLocator.updateconsumerlist`
:returns: None
"""
Updates the consumer list based on their availability.
recievedconsumerlist = self.communicator.discoveravailableconsumers()
if not recievedconsumerlist:
return
consumer_list = self.red.consumerlist
for recconsumer in recievedconsumerlist:
contains = False
for consumer in consumer_list:
if consumer["Host"] == recconsumer:
contains = True
if not contains:
LOGGER.info(f"Learned about new consumer at {recconsumer}")
consumer_list.append(
{"Host": recconsumer, "State": True, "LastOk": datetime.datetime.now().timestamp()})
self.red.consumerlist = consumer_list
self.updateconsumerlist()
def updateconsumerlist(self) -> None:
""" Updates the consumer list based on their availability.
Marks for each consumer if they are available or not. If a consumer is not available for some time (1 hour),
the it will be deleted from the list.
:return: None
"""
removeList = []
for consumer in self.consumerList:
removelist = []
consumer_list = self.red.consumerlist
for consumer in consumer_list:
if not self.communicator.checkconsumer(consumer["Host"]):
consumer["State"] = False
if datetime.datetime.now() - consumer["LastOk"] > datetime.timedelta(hours=1):
removeList.append(consumer)
if datetime.datetime.now() - datetime.datetime.fromtimestamp(consumer["LastOk"]) > datetime.timedelta(
seconds=15):
removelist.append(consumer)
else:
consumer["LastOk"] = datetime.datetime.now()
for rem in removeList:
self.consumerList.remove(rem)
consumer["LastOk"] = datetime.datetime.now().timestamp()
consumer["State"] = True
for rem in removelist:
consumer_list.remove(rem)
self.red.consumerlist = consumer_list
def updateconsumer(self):
"""
Checks all the consumers in the list and updates the current consumer if necessary.
"""If the current consumer is not available, checks all the consumers in the list and updates the current one.
Calls :func:`consumerlocator.ConsumerLocator.checkcurrentconsumer` and if needed
:func:`consumerlocator.ConsumerLocator.updateconsumerlist`. Sets the :class:`communicator.Communicator`
current instance with :func:`communicator.Communicator.set_currentconsumer`.
:return: the current consumer or None if there are no available customers at the moment.
"""
self.updateConsumerList()
if not self.checkConsumer():
if not self.checkcurrentconsumer():
self.updateconsumerlist()
newcurrentconsumer = None
newCurrentConsumer = None
for consumer in self.consumerList:
for consumer in self.red.consumerlist:
if consumer["State"]:
newCurrentConsumer = consumer
newcurrentconsumer = consumer
break
self.currentConsumer = newCurrentConsumer
self.red.currentconsumer = newcurrentconsumer
if self.red.currentconsumer is not None:
LOGGER.warning(f"Falling back to consumer at {newcurrentconsumer['Host']}")
self.learnconsumerlist()
if self.currentConsumer is not None:
return self.currentConsumer["Host"]
if self.red.currentconsumer is not None:
self.communicator.set_currentconsumer(self.red.currentconsumer["Host"])
return self.red.currentconsumer["Host"]
else:
return None
def getcurrentconsumer(self):
"""
Returns the currently selected consumer.
def getcurrentconsumer(self) -> str:
"""Returns the currently selected consumer's IP address.
:return: the current consumer
"""
return self.currentConsumer["Host"]
return self.red.currentconsumer["Host"]
def checkcurrentconsumer(self) -> bool:
"""Check the current consumer's health.
def checkcurrentconsumer(self):
"""
Check the consumers health.
:return: True if OK, False if fail
"""
if self.communicator.checkconsumer(self.currentConsumer["Host"]):
return True
else:
if self.red.currentconsumer is None:
return False
return self.communicator.checkconsumer(self.red.currentconsumer["Host"])

3
docs/Dockerfile Normal file
View File

@ -0,0 +1,3 @@
FROM httpd:2.4
COPY _build/html/ /usr/local/apache2/htdocs/

20
docs/Makefile Normal file
View File

@ -0,0 +1,20 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

54
docs/conf.py Normal file
View File

@ -0,0 +1,54 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('../'))
# -- Project information -----------------------------------------------------
project = 'Producer'
copyright = '2020, Torma Kristóf, Scharnitzky Donát, Kovács Bence'
author = 'Torma Kristóf, Scharnitzky Donát, Kovács Bence'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc'
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
autoclass_content = 'both'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

24
docs/index.rst Normal file
View File

@ -0,0 +1,24 @@
.. Producer documentation master file, created by
sphinx-quickstart on Tue Apr 7 17:01:40 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to Producer's documentation!
====================================
.. toctree::
:maxdepth: 2
:caption: Contents:
readme
source/modules
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
* `Git repository <https://git.kmlabz.com/GoldenPogacsa/producer>`_

35
docs/make.bat Normal file
View File

@ -0,0 +1,35 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd

2
docs/readme.rst Normal file
View File

@ -0,0 +1,2 @@
.. include:: ../README.rst

7
docs/source/app.rst Normal file
View File

@ -0,0 +1,7 @@
app module
==========
.. automodule:: app
:members:
:undoc-members:
:show-inheritance:

View File

@ -0,0 +1,7 @@
communicator module
===================
.. automodule:: communicator
:members:
:undoc-members:
:show-inheritance:

View File

@ -0,0 +1,7 @@
consumerlocator module
======================
.. automodule:: consumerlocator
:members:
:undoc-members:
:show-inheritance:

View File

@ -0,0 +1,7 @@
messagesender module
====================
.. automodule:: messagesender
:members:
:undoc-members:
:show-inheritance:

11
docs/source/modules.rst Normal file
View File

@ -0,0 +1,11 @@
producer
========
.. toctree::
:maxdepth: 4
app
communicator
consumerlocator
messagesender
test

7
docs/source/test.rst Normal file
View File

@ -0,0 +1,7 @@
test module
===========
.. automodule:: test
:members:
:undoc-members:
:show-inheritance:

87
integtest.py Normal file
View File

@ -0,0 +1,87 @@
#!/usr/bin/env python
"""
Integration test for the producer module.
"""
import re
from consumerinformation import ConsumerInformation
from communicator import Communicator
from consumerlocator import ConsumerLocator
from messagesender import MessageSender
from redisconnector import RedisConnector
from pytest_redis import factories
__author__ = "@dscharnitzky"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "integtest"
__version__text__ = "1"
generateduuid = '2fbff1f2-27e7-44c8-88d9-7348cee8c1c3'
redis_proc = factories.redis_proc(host='cache', port=6379)
redis_db = factories.redisdb('redis_nooproc')
answer = ""
def answermethod(Request):
global answer
answer = Request
return ""
def test_integration(httpserver):
"""
Tests the whole system.
:param httpserver: simple HTTP server
"""
#Inint
httpserver.expect_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["localhost", "localhost", "1.2.3.4"])
httpserver.expect_request(
uri="/log",
method='POST').respond_with_handler(answermethod)
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnector = RedisConnector()
consumerinfomation = ConsumerInformation(redisconnector=redisconnector)
comm = Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfomation)
messagesender = MessageSender(communicator=comm, uuid=generateduuid)
consumerlocator = ConsumerLocator(communicator=comm,
redisconnector=redisconnector)
#First test method
consumerlocator.learnconsumerlist()
conslist = consumerlocator.red.consumerlist
assert len(conslist) == 3
#Mock in port numbers
consrepllist = []
for consumer in conslist:
newconsumer = {"Host": consumer["Host"]+":"+port, "State": consumer["State"], "LastOk": consumer["LastOk"]}
consrepllist.append(newconsumer)
consumerlocator.red.consumerlist = consrepllist
conslist = consumerlocator.red.consumerlist
#Second test method call
consumerlocator.updateconsumerlist()
conslist = consumerlocator.red.consumerlist
error = False
for consumer in conslist:
if consumer["Host"] == "localhost:"+port and consumer["State"] is True:
pass
elif consumer["Host"] == "1.2.3.4:"+port and consumer["State"] is False:
pass
elif consumer["Host"] == "10.69.42.1:"+port and consumer["State"] is False:
pass
else:
error = True
assert error is False
#Third test method call
consumerlocator.updateconsumer()
assert consumerlocator.red.currentconsumer["Host"] == "localhost:"+port
messagesender.sendmessage()
assert answer != ""

View File

@ -1,48 +1,59 @@
#!/usr/bin/env python
"""
Message sender component.
"""
import logging
import random
import string
import json
from communicator import Communicator
"""
Main Flask RESTful API
"""
__author__ = "@tormakris"
__author__ = "@kovacsbence"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "messagesender"
__version__text__ = "1"
logging.basicConfig(level=logging.INFO)
LOGGER = logging.getLogger(__name__)
class MessageSender:
"""
Üzenetek küldéséért felelős komponens.
Component responsible for sending the messages. Requires an instance of :class:`communicator.Communicator`.
"""
def __init__(self):
"""
Inicializálja az osztályt.
"""
pass
def __init__(self, communicator: Communicator, uuid: str):
"""**Constructor:**
Initializes the object.
:param communicator: an instance of :class:`communicator.Communicator`.
"""
self.communicator = communicator
self.uuid = uuid
def randomString(self, stringLength):
"""Generate a random string of fixed length """
def randomstring(self, stringlength: int) -> str:
"""Generate a random string of fixed length
:param stringlength: the length of the string
:return: the generated string
"""
letters = string.ascii_lowercase
return ''.join(random.choice(letters) for i in range(stringLength))
return ''.join(random.choice(letters) for i in range(stringlength))
def sendmessage(self, message: str = "") -> None:
"""Sends the given message.
def createMessage(self, p) -> str:
If the message is omitted (empty), then a random message will be generated with length 23 (with
:func:`messagesender.MessageSender.randomstring`. Calls :func:`communicator.Communicator.sendmessage`
to send the message.
:param message: the message of type string that will be sent.
:return: None
"""
Ez egy metodus
:param szam:
:param szoveg:
:return: str tipus
"""
data = self.randomString(p)
return str(data)
pass
if __name__ == "__main__":
temp = MessageSender()
print(temp.createMessage(20))
if not message:
data = self.randomstring(32)
else:
data = message
self.communicator.sendmessage(json.dumps({"message": data, "uuid": self.uuid}))

63
redisconnector.py Normal file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python
import os
import redis
import json
"""
Redis interaction
"""
__author__ = "@tormakris"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "redis"
__version__text__ = "1"
REDISHOST = os.getenv("PRODUCER_REDIS", 'localhost')
class RedisConnector:
"""
Class abstracting Redis communication
"""
def __init__(self):
"""
Initialize class
"""
self.redisconnection = redis.StrictRedis(host=REDISHOST, port=6379, db=0)
def get_consumerlist(self):
"""
Gets list of consumers stored in Redis.
:return:
"""
return json.loads(self.redisconnection.get('consumerList'))
def set_consumerlist(self, consumerlist):
"""
Sets list of consumers stored in Redis.
:param consumerlist:
:return:
"""
json_list = json.dumps(consumerlist)
self.redisconnection.set('consumerList', json_list)
def get_currentconsumer(self):
"""
Gets currently active consumer.
:return:
"""
return json.loads(self.redisconnection.get('currentConsumer'))
def set_currentconsumer(self, currentconsumer):
"""
Sets currently active consumer
:param currentconsumer:
:return:
"""
json_dict = json.dumps(currentconsumer)
self.redisconnection.set('currentConsumer', json_dict)
consumerlist = property(get_consumerlist, set_consumerlist)
currentconsumer = property(get_currentconsumer, set_currentconsumer)

View File

@ -1,2 +1,7 @@
sentry_sdk
requests
pytest
pytest-mock
pytest-httpserver
pytest-redis
redis

View File

@ -1,30 +0,0 @@
#!/usr/bin/env python
"""
Singleton meta module
"""
__author__ = "@tormakris"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "singleton"
__version__text__ = "1"
class Singleton(object):
"""
Singleton metaclass
"""
_instances = {}
def __new__(class_, *args, **kwargs):
"""
New
:param args:
:param kwargs:
:return:
"""
if class_ not in class_._instances:
class_._instances[class_] = super(
Singleton, class_).__new__(
class_, *args, **kwargs)
return class_._instances[class_]

341
test.py Normal file
View File

@ -0,0 +1,341 @@
#!/usr/bin/env python
"""
Unit tests for the producer module.
"""
import re
import consumerlocator
import communicator
import messagesender
import redisconnector
import consumerinformation
from pytest_redis import factories
__author__ = "@tormakris"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "test"
__version__text__ = "1"
generateduuid = 'c959ad81-58f9-4445-aab4-8f3d68aee1ad'
redis_proc = factories.redis_proc(host='cache', port=6379)
redis_db = factories.redisdb('redis_nooproc')
def test_generate_string(mocker):
"""
Tests :func:`messagesender.MessageSender.randomstring`.
:param mocker: patches the :class:`communicator.Communicator`.
"""
mocker.patch('communicator.Communicator')
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
comm = communicator.Communicator(
currentconsumer="localhost",
uuid=generateduuid, consumerinformation=consumerinfo)
mess = messagesender.MessageSender(communicator=comm, uuid=generateduuid)
msg = mess.randomstring(stringlength=32)
assert isinstance(msg, str)
assert len(msg) == 32
def test_sendmessage(httpserver):
"""
Tests :func:`communicator.Communicator.sendmessage`.
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_oneshot_request(
uri="/log",
method='POST',
data="{\"uuid\": \"c959ad81-58f9-4445-aab4-8f3d68aee1ad\", \"message\": \"SENDING\"}").respond_with_json(
{
"test": "ok"})
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
comm = communicator.Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfo)
mess = "SENDING"
ret = comm.sendmessage(message=mess)
assert ret is None
def test_send_message(mocker):
"""
Tests :func:`messagesender.MessageSender.sendmessage`.
:param mocker: patches the :class:`communicator.Communicator`.
:return: None
"""
mocker.patch('communicator.Communicator')
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
redisconnect.currentconsumer = {"Host": f"127.0.0.1", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1", "State": True, "LastOk": "1589479202"}]
comm = communicator.Communicator(
currentconsumer="127.0.0.1",
uuid=generateduuid, consumerinformation=consumerinfo)
mess = messagesender.MessageSender(communicator=comm, uuid=generateduuid)
messa = "SENDING"
msg = mess.sendmessage(message=messa)
assert msg is None
def test_discoveravailableconsumers(httpserver):
"""
Tests :func:`communicator.Communicator.discoveravailableconsumers`
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_oneshot_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["10.69.42.1", "10.10.10.10", "10.20.30.40"])
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
comm = communicator.Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfo)
ret = comm.discoveravailableconsumers()
assert isinstance(ret, list)
assert ret == ["10.69.42.1", "10.10.10.10", "10.20.30.40"]
def test_isconsumeravailable(httpserver):
"""
Tests :func:`communicator.Communicator.isconsumeravailable`.
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_oneshot_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["10.69.42.1", "10.10.10.10", "10.20.30.40"])
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
comm = communicator.Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfo)
ret = comm.isconsumeravailable()
assert isinstance(ret, bool)
assert ret
ret2 = comm.isconsumeravailable()
assert isinstance(ret2, bool)
assert ret2 == False
comm2 = communicator.Communicator(
currentconsumer="127.0.0.1:69",
uuid=generateduuid, consumerinformation=consumerinfo)
ret3 = comm2.isconsumeravailable()
assert isinstance(ret3, bool)
assert ret3 == False
def test_checkconsumer(httpserver):
"""
Tests :func:`communicator.Communicator.checkconsumer`.
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_oneshot_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["10.69.42.1", "10.10.10.10", "10.20.30.40"])
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
comm = communicator.Communicator(
currentconsumer="127.0.0.1",
uuid=generateduuid, consumerinformation=consumerinfo)
ret = comm.checkconsumer(f"127.0.0.1:{port}")
assert isinstance(ret, bool)
assert ret
ret2 = comm.checkconsumer(f"127.0.0.1:{port}")
assert isinstance(ret2, bool)
assert ret2 == False
comm2 = communicator.Communicator(
currentconsumer="127.0.0.1",
uuid=generateduuid, consumerinformation=consumerinfo)
ret3 = comm2.checkconsumer(f"127.0.0.1:{port}")
assert isinstance(ret3, bool)
assert ret3 == False
def test_setcurrentconsumer():
"""
Tests :func:`communicator.Communicator.set_currentconsumer`
:return: None
"""
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
comm = communicator.Communicator(
currentconsumer="127.0.0.1",
uuid=generateduuid, consumerinformation=consumerinfo)
comm.set_currentconsumer("10.69.42.1")
assert comm.currenctconsumer == "10.69.42.1"
def test_learnconsumerlist(httpserver):
"""
Tests :func:`consumerlocator.ConsumerLocator.learnconsumerlist`
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["10.69.42.1", "10.10.10.10", "10.20.30.40"])
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
comm = communicator.Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfo)
consumerlocator.KNOWNCONSUMER = f"127.0.0.1:{port}"
locator = consumerlocator.ConsumerLocator(
communicator=comm,
redisconnector=redisconnector.RedisConnector())
ret = locator.learnconsumerlist()
assert ret is None
def test_getcurrentconsumer(mocker):
"""
Tests :func:`consumerlocator.ConsumerLocator.getcurrentconsumer`
:param mocker: patches the :class:`communicator.Communicator`.
:return: None
"""
mocker.patch('communicator.Communicator')
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
comm = communicator.Communicator(
currentconsumer="127.0.0.1",
uuid=generateduuid, consumerinformation=consumerinfo)
locator = consumerlocator.ConsumerLocator(
communicator=comm,
redisconnector=redisconnector.RedisConnector())
assert locator.getcurrentconsumer() == consumerlocator.KNOWNCONSUMER
def test_checkcurrentconsumer(httpserver):
"""
Tests :func:`consumerlocator.ConsumerLocator.checkcurrentconsumer`
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_oneshot_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["10.69.42.1", "10.10.10.10", "10.20.30.40"])
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
comm = communicator.Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfo)
consumerlocator.KNOWNCONSUMER = f"127.0.0.1:{port}"
locator = consumerlocator.ConsumerLocator(
communicator=comm,
redisconnector=redisconnector.RedisConnector())
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
ret = locator.checkcurrentconsumer()
assert ret == True
def test_updateconsumer(httpserver):
"""
Tests :func:`consumerlocator.ConsumerLocator.updateconsumer`
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_oneshot_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["10.69.42.1", "10.10.10.10", "10.20.30.40"])
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
comm = communicator.Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfo)
consumerlocator.KNOWNCONSUMER = f"127.0.0.1:{port}"
redisconn = redisconnector.RedisConnector()
locator = consumerlocator.ConsumerLocator(
communicator=comm,
redisconnector=redisconn)
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
assert redisconn.currentconsumer is not None
ret = locator.updateconsumer()
assert ret == f"127.0.0.1:{port}"
def test_updateconsumerlist(httpserver):
"""
Tests :func:`consumerlocator.ConsumerLocator.updateconsumerlist`
:param httpserver: simple HTTP server
:return: None
"""
httpserver.expect_oneshot_request(
uri="/consumers",
method='GET',
data="").respond_with_json(
["10.69.42.1", "10.10.10.10", "10.20.30.40"])
url = httpserver.url_for("/")
port = re.match(r"\W*http[^:]*\D*(\d+)", url).group(1)
redisconnect = redisconnector.RedisConnector()
consumerinfo = consumerinformation.ConsumerInformation(redisconnector=redisconnect)
comm = communicator.Communicator(
currentconsumer=f"127.0.0.1:{port}",
uuid=generateduuid, consumerinformation=consumerinfo)
consumerlocator.KNOWNCONSUMER = f"127.0.0.1:{port}"
locator = consumerlocator.ConsumerLocator(
communicator=comm,
redisconnector=redisconnector.RedisConnector())
redisconnect.currentconsumer = {"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}
redisconnect.consumerlist = [{"Host": f"127.0.0.1:{port}", "State": True, "LastOk": "1589479202"}]
ret = locator.updateconsumerlist()
assert ret is None