consumer-scheduler/app.py

#!/usr/bin/env python
import sentry_sdk
import time
import requests
import requests.exceptions
import os
import redis
import json
import logging
import socket

"""
Scheduler
"""

__author__ = "@kocsisr"
__copyright__ = "Copyright 2020, GoldenPogácsa Team"
__module_name__ = "app"
__version__text__ = "1"

sentry_sdk.init("https://0a106e104e114bc9a3fa47f9cb0db2f4@sentry.kmlabz.com/10")


def main():
    # set logging preferences
    logging.basicConfig(filename='', level=logging.DEBUG)

    # connect to redis
    r = redis.Redis(host='localhost', port=6379, db=0)

    # set initial consumer addresses
    ip_list = os.environ['INITIAL_SERVERS'].split(',')
    logging.debug('Get consumer list from environ at first: Done')
    # get the dictionary of the currently available consumers
    consumer_list_redis = json.loads((r.get('consumer_list') or b'{}').decode('utf-8'))
    logging.debug('Get consumer list from redis at first: Done')
    temp_dict = {}

    host_name = socket.gethostname()
    current_ip = socket.gethostbyname(host_name)

    for ip in ip_list:
        try:
            # request synchronization
            response = requests.post(f"http://{ip}/sync", json={'uuid': os.environ['LOCAL_UUID']})
        except requests.exceptions.ConnectionError:
            continue

        if response.status_code == 200:
            temp_dict[response.json()['uuid']] = {'ip': ip}

    consumer_list_redis.update(temp_dict)
    r.set('consumer_list', json.dumps(consumer_list_redis).encode('utf-8'))
    logging.debug('Update redis consumers ip list from first answers: Done')
    while True:
        logging.debug('Infinite Cycle start : Done')
        # get the dictionary of the currently available consumers
        consumer_list_redis = json.loads((r.get('consumer_list') or b'{}').decode('utf-8'))
        logging.debug('Get consumer list from redis: Done')
        for uuid, info in consumer_list_redis.items():
            ip = info['ip']
            try:
                # request synchronization
                response = requests.post(f"http://{ip}/sync", json={'uuid': os.environ['LOCAL_UUID']})
            except requests.exceptions.ConnectionError:
                continue

            if response.status_code == 200:
                temp_dict[response.json()['uuid']] = {'ip': ip}

        # update the dictionary of the currently available consumers
        consumer_list_redis.update(temp_dict)
        r.set('consumer_list', json.dumps(consumer_list_redis).encode('utf-8'))
        logging.debug('Update redis consumer ip list from answers: Done')

        logging.debug('Waiting for next turn')
        # wait for the next update time

        if current_ip != socket.gethostbyname(host_name):
            logging.debug('IP changed. Pushing updates...')
            current_ip = socket.gethostbyname(host_name)
            keys = r.keys('producer_*')

            logging.debug(f'Pushing update to the following producers: ' + ', '.join(k.decode('utf-8') for k in keys))

            for key in keys:
                ip = r.get(key)
                if ip:
                    ip = ip.decode('utf-8')
                else:
                    continue

                try:
                    response = requests.post(f"http://{ip}/ip", json={'uuid': os.environ['LOCAL_UUID'], 'ip': current_ip})
                    logging.debug(f"Pushed update to {key.decode('utf-8')} at {ip}. Response: {response.status_code}")
                except requests.exceptions.ConnectionError as e:
                    logging.warning(f"Could not push update to {key.decode('utf-8')}: {str(e)}")
                    continue
        else:
            logging.debug('IP unchanged.')

        time.sleep(30)


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        pass