fix(ingest): Fix Relay auth issues and add e2e event ingestion test (#578)

This is a long-needed test that tests the whole pipeline from Nginx, Relay, to Kafka, and Snuba. The final missing piece is testing the Symbolicator integration.

This PR is also a follow up to #576 as it didn't solve the Relay issues fully (the earlier fix was a coincidence or is not as reliable as it seemed).

Fixes #486 (finally?).

.travis.yml

@@ -14,7 +14,7 @@ script:
   - ./install.sh
   - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST
   - docker-compose up -d
-  - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done'
+  - printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done'
   - ./test.sh
 
 after_failure:

install.sh

@@ -19,17 +19,31 @@ RELAY_CONFIG_YML='relay/config.yml'
 RELAY_CREDENTIALS_JSON='relay/credentials.json'
 SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt'
 
+# Courtesy of https://stackoverflow.com/a/2183063/90297
+trap_with_arg() {
+    func="$1" ; shift
+    for sig ; do
+        trap "$func $sig" "$sig"
+    done
+}
+
 DID_CLEAN_UP=0
 # the cleanup function will be the exit point
 cleanup () {
   if [ "$DID_CLEAN_UP" -eq 1 ]; then
     return 0;
   fi
-  echo "Cleaning up..."
-  $dc stop &> /dev/null
   DID_CLEAN_UP=1
+
+  if [ "$1" != "EXIT" ]; then
+    echo "An error occurred, caught SIG$1";
+    echo "Cleaning up..."
+  fi
+
+  $dc stop &> /dev/null
 }
-trap cleanup ERR INT TERM
+trap_with_arg cleanup ERR INT TERM EXIT
+
 
 echo "Checking minimum requirements..."
 
@@ -246,9 +260,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then
   echo "Relay credentials written to $RELAY_CREDENTIALS_JSON"
 fi
 
-
-cleanup
-
 echo ""
 echo "----------------"
 echo "You're all done! Run the following command to get Sentry running:"

sentry/sentry.conf.example.py

@@ -12,22 +12,22 @@ def get_internal_network():
     import socket
     import struct
 
-    iface = 'eth0'
+    iface = "eth0"
     sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    ifreq = struct.pack('16sH14s', iface, socket.AF_INET, b'\x00' * 14)
+    ifreq = struct.pack("16sH14s", iface, socket.AF_INET, b"\x00" * 14)
 
     try:
         ip = struct.unpack(
-            "!I", struct.unpack('16sH2x4s8x', fcntl.ioctl(sockfd, 0x8915, ifreq))[2]
+            "!I", struct.unpack("16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2]
         )[0]
         netmask = socket.ntohl(
-            struct.unpack('16sH2xI8x', fcntl.ioctl(sockfd, 0x891B, ifreq))[2]
+            struct.unpack("16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2]
         )
     except IOError:
         return ()
     base = socket.inet_ntoa(struct.pack("!I", ip & netmask))
     netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1))
-    return ('{0:s}/{1:d}'.format(base, netmask_bits),)
+    return ("{0:s}/{1:d}".format(base, netmask_bits),)
 
 
 INTERNAL_IPS = get_internal_network()
@@ -60,7 +60,7 @@ SENTRY_USE_BIG_INTS = True
 SENTRY_SINGLE_ORGANIZATION = True
 
 SENTRY_OPTIONS["system.event-retention-days"] = int(
-    env('SENTRY_EVENT_RETENTION_DAYS', '90')
+    env("SENTRY_EVENT_RETENTION_DAYS", "90")
 )
 
 #########
@@ -185,32 +185,33 @@ SENTRY_DIGESTS = "sentry.digests.backends.redis.RedisBackend"
 SENTRY_WEB_HOST = "0.0.0.0"
 SENTRY_WEB_PORT = 9000
 SENTRY_WEB_OPTIONS = {
-    "http-keepalive": True,
+    "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT),
+    "protocol": "uwsgi",
+    # This is need to prevent https://git.io/fj7Lw
+    "uwsgi-socket": None,
     "so-keepalive": True,
-    "http-auto-chunked": True,
+    # Keep this between 15s-75s as that's what Relay supports
+    "http-keepalive": 15,
     "http-chunked-input": True,
     # the number of web workers
-    'workers': 3,
+    "workers": 3,
-    'threads': 4,
+    "threads": 4,
-    # Turn off memory reporting
     "memory-report": False,
     # Some stuff so uwsgi will cycle workers sensibly
-    'max-requests': 100000,
+    "max-requests": 100000,
-    'max-requests-delta': 500,
+    "max-requests-delta": 500,
-    'max-worker-lifetime': 86400,
+    "max-worker-lifetime": 86400,
     # Duplicate options from sentry default just so we don't get
     # bit by sentry changing a default value that we depend on.
-    'thunder-lock': True,
+    "thunder-lock": True,
-    'log-x-forwarded-for': False,
+    "log-x-forwarded-for": False,
-    'buffer-size': 32768,
+    "buffer-size": 32768,
-    # Relay cannot authenticate without the following
+    "limit-post": 209715200,
-    'post-buffering': 32768,
+    "disable-logging": True,
-    'limit-post': 209715200,
+    "reload-on-rss": 600,
-    'disable-logging': True,
+    "ignore-sigpipe": True,
-    'reload-on-rss': 600,
+    "ignore-write-errors": True,
-    'ignore-sigpipe': True,
+    "disable-write-exception": True,
-    'ignore-write-errors': True,
-    'disable-write-exception': True,
 }
 
 ###########
@@ -259,7 +260,7 @@ SENTRY_FEATURES.update(
 # GitHub Integration #
 ######################
 
-GITHUB_EXTENDED_PERMISSIONS = ['repo']
+GITHUB_EXTENDED_PERMISSIONS = ["repo"]
 
 #########################
 # Bitbucket Integration #

test.sh

@@ -5,30 +5,91 @@ SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}"
 TEST_USER='test@example.com'
 TEST_PASS='test123TEST'
 COOKIE_FILE=$(mktemp)
-declare -a TEST_STRINGS=(
-    '"isAuthenticated":true'
-    '"username":"test@example.com"'
-    '"isSuperuser":true'
-)
 
-INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective})
+# Courtesy of https://stackoverflow.com/a/2183063/90297
-if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then
+trap_with_arg() {
+    func="$1" ; shift
+    for sig ; do
+        trap "$func $sig" "$sig"
+    done
+}
+
+DID_CLEAN_UP=0
+# the cleanup function will be the exit point
+cleanup () {
+  if [ "$DID_CLEAN_UP" -eq 1 ]; then
+    return 0;
+  fi
+  DID_CLEAN_UP=1
+
+  if [ "$1" != "EXIT" ]; then
+    echo "An error occurred, caught SIG$1";
+  fi
+
+  echo "Cleaning up..."
+  rm $COOKIE_FILE
+  echo "Done."
+}
+trap_with_arg cleanup ERR INT TERM EXIT
+
+get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; }
+sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; }
+
+login () {
+    INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective})
+    if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then
         echo "Initial /auth/login/ redirect failed, exiting..."
         echo "$INITIAL_AUTH_REDIRECT"
         exit -1
-fi
+    fi
 
-CSRF_TOKEN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" '
+    CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" '
         /csrfmiddlewaretoken/ {
         print $4 "=" $6;
         exit;
     }')
-LOGIN_RESPONSE=$(curl -sL -F 'op=login' -F "username=$TEST_USER" -F "password=$TEST_PASS" -F "$CSRF_TOKEN" "$SENTRY_TEST_HOST/auth/login/" -H "Referer: $SENTRY_TEST_HOST/auth/login/" -b "$COOKIE_FILE" -c "$COOKIE_FILE")
 
-TEST_RESULT=0
+    curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE";
-for i in "${TEST_STRINGS[@]}"
+}
+
+LOGIN_RESPONSE=$(login);
+declare -a LOGIN_TEST_STRINGS=(
+    '"isAuthenticated":true'
+    '"username":"test@example.com"'
+    '"isSuperuser":true'
+)
+for i in "${LOGIN_TEST_STRINGS[@]}"
 do
    echo "Testing '$i'..."
    echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null
    echo "Pass."
 done
+
+# Set up initial/required settings (InstallWizard request)
+sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null
+
+SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }')
+
+TEST_EVENT_ID=$(docker run --rm --net host -e "SENTRY_DSN=$SENTRY_DSN" -v $(pwd):/work getsentry/sentry-cli send-event -m "a failure" -e task:create-user -e object:42 | tr -d '-')
+echo "Created event $TEST_EVENT_ID."
+
+EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/"
+export -f sentry_api_request get_csrf_token
+export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH
+printf "Checking its existence"
+timeout 15 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done'
+echo "";
+
+EVENT_RESPONSE=$(sentry_api_request "projects/sentry/internal/events/$TEST_EVENT_ID/")
+declare -a EVENT_TEST_STRINGS=(
+    '"eventID":"'"$TEST_EVENT_ID"'"'
+    '"message":"a failure"'
+    '"title":"a failure"'
+    '"object":"42"'
+)
+for i in "${EVENT_TEST_STRINGS[@]}"
+do
+   echo "Testing '$i'..."
+   echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null
+   echo "Pass."
+done