add all services that have no secret stuff

.drone.yml

@@ -0,0 +1,44 @@
+---
+kind: pipeline
+type: docker
+name: ansible
+
+steps:
+- name: setup environment
+  image: ubuntu
+  environment:
+    SSH_KEY:
+      from_secret: SSH_KEY
+  commands:
+    - echo "$PWD"
+    - echo "$SSH_KEY" > $PWD/id_rsa && chmod 0600 $PWD/id_rsa
+
+- name: check ansible syntax
+  image: plugins/ansible:3
+  settings:
+    playbook: nightly.yaml
+    galaxy: requirements.yaml
+    inventory: inventory.yaml
+    syntax_check: true
+
+- name: run playbook in check mode
+  image: plugins/ansible:3
+  environment:
+    ANSIBLE_HOST_KEY_CHECKING: "False"
+    ANSIBLE_PRIVATE_KEY_FILE: "/drone/src/id_rsa"
+  settings:
+    playbook: nightly.yaml
+    galaxy: requirements.yaml
+    inventory: inventory.yaml
+    check: true
+
+- name: ansible nightly run
+  image: plugins/ansible:3
+  environment:
+    ANSIBLE_HOST_KEY_CHECKING: "False"
+    ANSIBLE_PRIVATE_KEY_FILE: "/drone/src/id_rsa"
+  settings:
+    playbook: nightly.yaml
+    galaxy: requirements.yaml
+    inventory: inventory.yaml
+...

ansible.cfg

@@ -0,0 +1,5 @@
+[ssh_connection]
+ssh_args = -o ControlMaster=auto -o ControlPersist=60s
+[defaults]
+forks=2
+pipelining = True

deploy.yaml

@@ -0,0 +1,31 @@
+---
+- name: "Deploy prometheus configuration"
+  hosts: monitoring
+  roles:
+    - prometheus
+
+- name: "Deploy matrix configuration"
+  hosts: matrix
+  roles:
+    - matrix
+
+- name: "Deploy nexus configuration"
+  hosts: nexus
+  roles:
+    - nexus
+
+- name: "Deploy swagger configuration and update"
+  hosts: swagger
+  roles:
+    - swagger
+
+- name: "Deploy drone configuration and update"
+  hosts: drone
+  roles:
+    - drone
+
+- name: "Update services automatically"
+  hosts: commonupdate
+  roles:
+    - update
+...

inventory.yaml

@@ -0,0 +1,16 @@
+---
+all:
+  children:
+    commonupdate:
+      hosts:
+        matrix:
+        nexus:
+        git:
+        monitoring:
+        guacamole:
+        nextcloud:
+    others:
+      hosts:
+        drone:
+        swagger:
+...

nightly.yaml

@@ -0,0 +1,26 @@
+---
+- name: "Deploy prometheus configuration"
+  hosts: monitoring
+  roles:
+    - prometheus
+
+- name: "Deploy matrix configuration"
+  hosts: matrix
+  roles:
+    - matrix
+
+- name: "Deploy nexus configuration"
+  hosts: nexus
+  roles:
+    - nexus
+
+- name: "Deploy swagger configuration and update"
+  hosts: swagger
+  roles:
+    - swagger
+
+- name: "Update services automatically"
+  hosts: commonupdate
+  roles:
+    - update
+...

requirements.yaml

@@ -0,0 +1,5 @@
+---
+collections:
+- ansible.posix
+- community.general
+...

roles/drone/tasks/main.yaml

@@ -0,0 +1,22 @@
+---
+- name: Update service
+  ansible.builtin.command: docker-compose pull
+  become: True
+  become_user: drone
+  args:
+    chdir: /home/drone
+
+- name: Shut down service
+  ansible.builtin.command: docker-compose down
+  become: True
+  become_user: drone
+  args:
+    chdir: /home/drone
+
+- name: Start service
+  ansible.builtin.command: docker-compose up -d
+  become: True
+  become_user: drone
+  args:
+    chdir: /home/drone
+...

roles/matrix/files/docker-compose.yaml

@@ -0,0 +1,40 @@
+---
+version: '3.2'
+services:
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    ports:
+      - 4194:8080
+    restart: always
+    networks:
+      - matrix
+
+  element:
+    image: vectorim/element-web:latest
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:8181:80"
+    volumes:
+      - ./element-config.json:/app/config.json
+    networks:
+      - matrix
+
+  synapse:
+    image: matrixdotorg/synapse:latest
+    restart: unless-stopped
+    networks:
+      - matrix
+    volumes:
+      - ./synapse:/data
+    ports:
+      - "127.0.0.1:8080:8080"
+
+networks:
+  matrix:
+    external: false
+...

roles/matrix/tasks/main.yaml

@@ -0,0 +1,9 @@
+---
+- name: Copy docker-compose configuration
+  ansible.builtin.copy:
+    src: docker-compose.yaml
+    dest: /home/service-user/docker-compose.yaml
+    owner: service-user
+    group: service-user
+    mode: '0644'
+...

roles/nexus/files/docker-compose.yaml

@@ -0,0 +1,31 @@
+---
+version: '3.4'
+
+networks:
+  nexus:
+    external: false
+services:
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    ports:
+      - 4194:8080
+    restart: always
+    networks:
+      - nexus
+
+  nexus:
+    image: sonatype/nexus3
+    networks:
+      - nexus
+    volumes:
+      - /mnt/data/nexusdata:/nexus-data
+    restart: always
+    ports:
+      - "127.0.0.1:8080:8081"
+      - "127.0.0.1:4269:4269"
+...

roles/nexus/tasks/main.yaml

@@ -0,0 +1,9 @@
+---
+- name: Copy docker-compose configuration
+  ansible.builtin.copy:
+    src: docker-compose.yaml
+    dest: /home/service-user/docker-compose.yaml
+    owner: service-user
+    group: service-user
+    mode: '0644'
+...

roles/prometheus/files/alert.rules

@@ -0,0 +1,13 @@
+groups:
+- name: example
+  rules:
+
+  # Alert for any instance that is unreachable for >2 minutes.
+  - alert: service_down
+    expr: up == 0
+    for: 2m
+    labels:
+      severity: page
+    annotations:
+      summary: "Instance {{ $labels.instance }} down"
+      description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."

roles/prometheus/files/alertmanager.yml

@@ -0,0 +1,13 @@
+---
+route:
+  receiver: 'email'
+
+receivers:
+  - name: 'email'
+    email_configs:
+      - to: "tormakristof@tormakristof.eu"
+        from: "monitoring@tormakris.dev"
+        smarthost: "smtp.intra.tormakris.dev:25"
+        tls_config:
+          insecure_skip_verify: true
+...

roles/prometheus/files/docker-compose.yaml

@@ -0,0 +1,80 @@
+---
+version: '3.7'
+
+networks:
+  monitoring:
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus/:/etc/prometheus/
+      - ./prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+#    ports:
+#      - "127.0.0.1:8080:9090"
+    depends_on:
+      - node-exporter
+      - cadvisor
+    networks:
+      - monitoring
+    restart: always
+
+  node-exporter:
+    image: prom/node-exporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.sysfs=/host/sys'
+      - --collector.filesystem.ignored-mount-points
+      - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
+    networks:
+      - monitoring
+    restart: always
+
+  alertmanager:
+    image: prom/alertmanager
+#    ports:
+#      - "127.0.0.1:8282:9093"
+    volumes:
+      - ./alertmanager/:/etc/alertmanager/
+    networks:
+      - monitoring
+    restart: always
+    command:
+      - '--config.file=/etc/alertmanager/config.yml'
+      - '--storage.path=/alertmanager'
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    networks:
+      - monitoring
+    restart: always
+
+  grafana:
+    image: grafana/grafana
+    user: "472"
+    depends_on:
+      - prometheus
+    ports:
+      - "127.0.0.1:8181:3000"
+    volumes:
+      - ./grafana_data:/var/lib/grafana
+    env_file:
+      - ./grafana/config.monitoring
+    networks:
+      - monitoring
+    restart: always
+...

roles/prometheus/files/prometheus.yml

@@ -0,0 +1,158 @@
+---
+# my global config
+global:
+  scrape_interval:     15s # By default, scrape targets every 15 seconds.
+  evaluation_interval: 15s # By default, scrape targets every 15 seconds.
+  # scrape_timeout is set to the global default (10s).
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+      monitor: 'stargate-cluster'
+
+# Load and evaluate rules in this file every 'evaluation_interval' seconds.
+rule_files:
+  - 'alert.rules'
+  # - "first.rules"
+  # - "second.rules"
+
+# alert
+alerting:
+  alertmanagers:
+  - scheme: http
+    static_configs:
+    - targets:
+      - "alertmanager:9093"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+
+  - job_name: 'prometheus'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+         - targets: ['localhost:9090']
+
+  - job_name: 'node-exporter'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['node-exporter:9100']
+      - targets: ['zelenka.intra.tormakris.dev:9100']
+      - targets: ['drone.intra.tormakris.dev:9100']
+      - targets: ['matrix.intra.tormakris.dev:9100']
+      - targets: ['swagger.intra.tormakris.dev:9100']
+      - targets: ['drone-runner.intra.tormakris.dev:9100']
+      - targets: ['smtp.intra.tormakris.dev:9100']
+      - targets: ['webgateway.intra.tormakris.dev:9100']
+      - targets: ['openvpn.intra.tormakris.dev:9100']
+      - targets: ['nexus.intra.tormakris.dev:9100']
+      - targets: ['git.intra.tormakris.dev:9100']
+      - targets: ['postgres.intra.tormakris.dev:9100']
+      - targets: ['guacamole.intra.tormakris.dev:9100']
+      - targets: ['bitwarden.intra.tormakris.dev:9100']
+      - targets: ['nextcloud.intra.tormakris.dev:9100']
+      - targets: ['backup.intra.tormakris.dev:9100']
+      - targets: ['ssh.intra.tormakris.dev:9100']
+      - targets: ['minecraft.intra.tormakris.dev:9100']
+
+  - job_name: 'postfix-exporter'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['zelenka.intra.tormakris.dev:9154']
+      - targets: ['drone.intra.tormakris.dev:9154']
+      - targets: ['matrix.intra.tormakris.dev:9154']
+      - targets: ['swagger.intra.tormakris.dev:9154']
+      - targets: ['drone-runner.intra.tormakris.dev:9154']
+      - targets: ['smtp.intra.tormakris.dev:9154']
+      - targets: ['webgateway.intra.tormakris.dev:9154']
+      - targets: ['openvpn.intra.tormakris.dev:9154']
+      - targets: ['nexus.intra.tormakris.dev:9154']
+      - targets: ['git.intra.tormakris.dev:9154']
+      - targets: ['guacamole.intra.tormakris.dev:9154']
+      - targets: ['bitwarden.intra.tormakris.dev:9154']
+      - targets: ['nextcloud.intra.tormakris.dev:9154']
+      - targets: ['backup.intra.tormakris.dev:9154']
+
+  - job_name: 'nginx-exporter'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['drone.intra.tormakris.dev:9113']
+      - targets: ['matrix.intra.tormakris.dev:9113']
+      - targets: ['swagger.intra.tormakris.dev:9113']
+      - targets: ['webgateway.intra.tormakris.dev:9113']
+      - targets: ['nexus.intra.tormakris.dev:9113']
+      - targets: ['git.intra.tormakris.dev:9113']
+      - targets: ['guacamole.intra.tormakris.dev:9113']
+      - targets: ['bitwarden.intra.tormakris.dev:9113']
+      - targets: ['nextcloud.intra.tormakris.dev:9113']
+
+  - job_name: 'cadvisor'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['cadvisor:8080']
+      - targets: ['zelenka.intra.tormakris.dev:4194']
+      - targets: ['drone.intra.tormakris.dev:4194']
+      - targets: ['matrix.intra.tormakris.dev:4194']
+      - targets: ['swagger.intra.tormakris.dev:4194']
+      - targets: ['drone-runner.intra.tormakris.dev:4194']
+      - targets: ['nexus.intra.tormakris.dev:4194']
+      - targets: ['git.intra.tormakris.dev:4194']
+      - targets: ['guacamole.intra.tormakris.dev:4194']
+      - targets: ['nextcloud.intra.tormakris.dev:4194']
+
+  - job_name: 'drone-server'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['drone.intra.tormakris.dev:443']
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+    bearer_token: a96fdbbbfb1072836bf81b2eab456773
+
+  - job_name: 'postgres-exporter'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['postgresql.intra.tormakris.dev:9187']
+
+  - job_name: 'gitea-server'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['git.intra.tormakris.dev:443']
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+
+  - job_name: 'windows_exporter'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+
+    static_configs:
+      - targets: ['woolsey-host.intra.tormakris.dev:9182']
+      - targets: ['mckay-host.intra.tormakris.dev:9182']
+...

roles/prometheus/tasks/main.yaml

@@ -0,0 +1,33 @@
+---
+- name: Copy prometheus configuration
+  ansible.builtin.copy:
+    src: prometheus.yml
+    dest: /home/service-user/prometheus/prometheus.yml
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Copy alertmanager configuration
+  ansible.builtin.copy:
+    src: alertmanager.yml
+    dest: /home/service-user/alertmanager/config.yml
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Copy alert rules
+  ansible.builtin.copy:
+    src: alert.rules
+    dest: /home/service-user/prometheus/alert.rules
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Copy docker-compose configuration
+  ansible.builtin.copy:
+    src: docker-compose.yaml
+    dest: /home/service-user/docker-compose.yaml
+    owner: service-user
+    group: service-user
+    mode: '0644'
+...

roles/swagger/files/docker-compose.yml

@@ -0,0 +1,30 @@
+---
+version: '3'
+networks:
+  swagger:
+    external: false
+
+services:
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    ports:
+      - 4194:8080
+    restart: always
+    networks:
+      - swagger
+
+  swagger:
+    image: swaggerapi/swagger-ui
+    restart: always
+    ports:
+      - "127.0.0.1:8080:8080"
+    environment:
+      URLS: "[ { name: \"Input Service\", url: \"https://git.kmlabz.com/birbnetes/swagger-docs/raw/branch/master/input.yml\"}, { name: \"Storage Service\", url: \"https://git.kmlabz.com/birbnetes/swagger-docs/raw/branch/master/storage.yml\" }, {name: \"Output Service (RDB)\", url: \"https://git.kmlabz.com/birbnetes/swagger-docs/raw/branch/master/output-service-rdb.yml\"}, {name: \"Model Service\", url: \"https://git.kmlabz.com/birbnetes/swagger-docs/raw/branch/master/model.yaml\"}, {name: \"Command and Control\", url: \"https://git.kmlabz.com/birbnetes/swagger-docs/raw/branch/master/command-and-control.yml\"}, { name: \"Service Locator\", url: \"https://git.kmlabz.com/dnsproject/swagger/raw/branch/master/servicelocator.yml\"} ]"
+    networks:
+      - swagger
+...

roles/swagger/tasks/main.yaml

@@ -0,0 +1,30 @@
+---
+- name: Copy docker-compose configuration
+  ansible.builtin.copy:
+    src: docker-compose.yaml
+    dest: /home/swagger/docker-compose.yaml
+    owner: swagger
+    group: swagger
+    mode: '0644'
+
+- name: Update service
+  ansible.builtin.command: docker-compose pull
+  become: True
+  become_user: swagger
+  args:
+    chdir: /home/swagger
+
+- name: Shut down service
+  ansible.builtin.command: docker-compose down
+  become: True
+  become_user: swagger
+  args:
+    chdir: /home/swagger
+
+- name: Start service
+  ansible.builtin.command: docker-compose up -d
+  become: True
+  become_user: swagger
+  args:
+    chdir: /home/swagger
+...

roles/update/tasks/main.yaml

@@ -0,0 +1,22 @@
+---
+- name: Update service
+  ansible.builtin.command: docker-compose pull
+  become: True
+  become_user: service-user
+  args:
+    chdir: /home/service-user
+
+- name: Shut down service
+  ansible.builtin.command: docker-compose down
+  become: True
+  become_user: service-user
+  args:
+    chdir: /home/service-user
+
+- name: Start service
+  ansible.builtin.command: docker-compose up -d
+  become: True
+  become_user: service-user
+  args:
+    chdir: /home/service-user
+...

run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ansible-galaxy collection install -r requirements.yaml
+ansible-playbook --ask-become-pass --ask-pass -i inventory.yaml $1
+#ansible-playbook --ask-become-pass -i inventory.yaml $1