feat(relay): Add Relay to onpremise installation (#421)

Co-Authored-By: Burak Yigit Kaya <byk@sentry.io>

.gitignore

@@ -77,3 +77,4 @@ data/
 sentry/sentry.conf.py
 sentry/config.yml
 sentry/requirements.txt
+relay/credentials.json

docker-compose.yml

@@ -141,14 +141,15 @@ services:
       - 'sentry-symbolicator:/data'
   web:
     << : *sentry_defaults
-    ports:
-      - '9000:9000/tcp'
   cron:
     << : *sentry_defaults
     command: run cron
   worker:
     << : *sentry_defaults
     command: run worker
+  ingest-consumer:
+    << : *sentry_defaults
+    command: run ingest-consumer --all-consumer-types
   post-process-forwarder:
     << : *sentry_defaults
     # Increase `--commit-batch-size 1` below to deal with high-load environments.
@@ -161,6 +162,28 @@ services:
       args:
         BASE_IMAGE: 'sentry-onpremise-local'
     command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"'
+  nginx:
+    ports:
+      - '9000:80/tcp'
+    image: "nginx:1.16"
+    volumes:
+      - type: bind
+        read_only: true
+        source: ./nginx
+        target: /etc/nginx
+    depends_on:
+      - web
+      - relay
+  relay:
+    image: "us.gcr.io/sentryio/relay:latest"
+    command: 'run --config /etc/relay'
+    volumes:
+      - type: bind
+        source: ./relay
+        target: /etc/relay
+    depends_on:
+      - kafka
+      - redis
 volumes:
   sentry-data:
     external: true

install.sh

@@ -14,6 +14,8 @@ MIN_RAM=2400 # MB
 
 SENTRY_CONFIG_PY='sentry/sentry.conf.py'
 SENTRY_CONFIG_YML='sentry/config.yml'
+RELAY_CONFIG_YML='relay/config.yml'
+RELAY_CREDENTIALS_JSON='relay/credentials.json'
 SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt'
 
 DID_CLEAN_UP=0
@@ -171,6 +173,28 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then
     "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data"
 fi
 
+
+if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then
+    echo ""
+    echo "Generating Relay credentials..."
+
+    $dcr --user $(id -u) relay --config /etc/relay credentials generate --overwrite
+    chmod a+r $RELAY_CREDENTIALS_JSON
+    CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON")
+    CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]"
+
+    if grep -xq SENTRY_RELAY_WHITELIST_PK "$SENTRY_CONFIG_PY"; then
+        >&2 echo "FAIL: SENTRY_RELAY_WHITELIST_PK already exists in $SENTRY_CONFIG_PY, please replace with:"
+        >&2 echo ""
+        >&2 echo "  $CREDENTIALS"
+        >&2 echo ""
+        exit 1
+    fi
+
+     echo "" >> "$SENTRY_CONFIG_PY"
+     echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY"
+fi
+
 cleanup
 
 echo ""

nginx/nginx.conf

@@ -0,0 +1,47 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    sendfile        on;
+
+    keepalive_timeout  65;
+
+    upstream relay {
+        server relay:3000;
+    }
+
+    upstream sentry {
+        server web:9000;
+    }
+
+    server {
+        listen 80;
+        # use the docker DNS server to resolve ips for relay and sentry containers
+        resolver 127.0.0.11 ipv6=off;
+        client_max_body_size 100M;
+        location /api/store/ {
+            proxy_pass http://relay;
+        }
+        location ~ ^/api/[1-9]\d*/ {
+            proxy_pass http://relay;
+        }
+        location / {
+           proxy_pass http://sentry;
+        }
+    }
+}

relay/config.yml

@@ -0,0 +1,13 @@
+---
+relay:
+  upstream: "http://web:9000/"
+  host: 0.0.0.0
+  port: 3000
+#logging:
+#  # Available logging levels: TRACE, DEBUG, INFO, WARN, ERROR
+#  level: WARN
+processing:
+  enabled: true
+  kafka_config:
+    - {name: "bootstrap.servers", value: "kafka:9092"}
+  redis: redis://redis:6379

sentry/sentry.conf.example.py

@@ -220,3 +220,4 @@ GITHUB_EXTENDED_PERMISSIONS = ['repo']
 
 # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY'
 # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET'
+