fix(ingest): Fix Relay auth issues and add e2e event ingestion test (#578)

This is a long-needed test that tests the whole pipeline from Nginx, Relay, to Kafka, and Snuba. The final missing piece is testing the Symbolicator integration. This PR is also a follow up to #576 as it didn't solve the Relay issues fully (the earlier fix was a coincidence or is not as reliable as it seemed). Fixes #486 (finally?).
2020-07-13 13:07:05 +03:00 · 2020-07-13 13:07:05 +03:00 · 75fe6c073b
commit 75fe6c073b
parent 1c9bfd9017
4 changed files with 124 additions and 51 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -14,7 +14,7 @@ script:
  - ./install.sh
  - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST
  - docker-compose up -d
-  - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done'
+  - printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done'
  - ./test.sh

 after_failure:
--- a/install.sh
+++ b/install.sh
@ -19,17 +19,31 @@ RELAY_CONFIG_YML='relay/config.yml'
 RELAY_CREDENTIALS_JSON='relay/credentials.json'
 SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt'

+# Courtesy of https://stackoverflow.com/a/2183063/90297
+trap_with_arg() {
+    func="$1" ; shift
+    for sig ; do
+        trap "$func $sig" "$sig"
+    done
+}
+
 DID_CLEAN_UP=0
 # the cleanup function will be the exit point
 cleanup () {
  if [ "$DID_CLEAN_UP" -eq 1 ]; then
    return 0;
  fi
-  echo "Cleaning up..."
-  $dc stop &> /dev/null
  DID_CLEAN_UP=1
+
+  if [ "$1" != "EXIT" ]; then
+    echo "An error occurred, caught SIG$1";
+    echo "Cleaning up..."
+  fi
+
+  $dc stop &> /dev/null
 }
-trap cleanup ERR INT TERM
+trap_with_arg cleanup ERR INT TERM EXIT
+

 echo "Checking minimum requirements..."

@ -246,9 +260,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then
  echo "Relay credentials written to $RELAY_CREDENTIALS_JSON"
 fi

-
-cleanup
-
 echo ""
 echo "----------------"
 echo "You're all done! Run the following command to get Sentry running:"
--- a/sentry/sentry.conf.example.py
+++ b/sentry/sentry.conf.example.py
@ -12,22 +12,22 @@ def get_internal_network():
    import socket
    import struct

-    iface = 'eth0'
+    iface = "eth0"
    sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    ifreq = struct.pack('16sH14s', iface, socket.AF_INET, b'\x00' * 14)
+    ifreq = struct.pack("16sH14s", iface, socket.AF_INET, b"\x00" * 14)

    try:
        ip = struct.unpack(
-            "!I", struct.unpack('16sH2x4s8x', fcntl.ioctl(sockfd, 0x8915, ifreq))[2]
+            "!I", struct.unpack("16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2]
        )[0]
        netmask = socket.ntohl(
-            struct.unpack('16sH2xI8x', fcntl.ioctl(sockfd, 0x891B, ifreq))[2]
+            struct.unpack("16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2]
        )
    except IOError:
        return ()
    base = socket.inet_ntoa(struct.pack("!I", ip & netmask))
    netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1))
-    return ('{0:s}/{1:d}'.format(base, netmask_bits),)
+    return ("{0:s}/{1:d}".format(base, netmask_bits),)


 INTERNAL_IPS = get_internal_network()
@ -60,7 +60,7 @@ SENTRY_USE_BIG_INTS = True
 SENTRY_SINGLE_ORGANIZATION = True

 SENTRY_OPTIONS["system.event-retention-days"] = int(
-    env('SENTRY_EVENT_RETENTION_DAYS', '90')
+    env("SENTRY_EVENT_RETENTION_DAYS", "90")
 )

 #########
@ -185,32 +185,33 @@ SENTRY_DIGESTS = "sentry.digests.backends.redis.RedisBackend"
 SENTRY_WEB_HOST = "0.0.0.0"
 SENTRY_WEB_PORT = 9000
 SENTRY_WEB_OPTIONS = {
-    "http-keepalive": True,
+    "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT),
+    "protocol": "uwsgi",
+    # This is need to prevent https://git.io/fj7Lw
+    "uwsgi-socket": None,
    "so-keepalive": True,
-    "http-auto-chunked": True,
+    # Keep this between 15s-75s as that's what Relay supports
+    "http-keepalive": 15,
    "http-chunked-input": True,
    # the number of web workers
-    'workers': 3,
-    'threads': 4,
-    # Turn off memory reporting
+    "workers": 3,
+    "threads": 4,
    "memory-report": False,
    # Some stuff so uwsgi will cycle workers sensibly
-    'max-requests': 100000,
-    'max-requests-delta': 500,
-    'max-worker-lifetime': 86400,
+    "max-requests": 100000,
+    "max-requests-delta": 500,
+    "max-worker-lifetime": 86400,
    # Duplicate options from sentry default just so we don't get
    # bit by sentry changing a default value that we depend on.
-    'thunder-lock': True,
-    'log-x-forwarded-for': False,
-    'buffer-size': 32768,
-    # Relay cannot authenticate without the following
-    'post-buffering': 32768,
-    'limit-post': 209715200,
-    'disable-logging': True,
-    'reload-on-rss': 600,
-    'ignore-sigpipe': True,
-    'ignore-write-errors': True,
-    'disable-write-exception': True,
+    "thunder-lock": True,
+    "log-x-forwarded-for": False,
+    "buffer-size": 32768,
+    "limit-post": 209715200,
+    "disable-logging": True,
+    "reload-on-rss": 600,
+    "ignore-sigpipe": True,
+    "ignore-write-errors": True,
+    "disable-write-exception": True,
 }

 ###########
@ -259,7 +260,7 @@ SENTRY_FEATURES.update(
 # GitHub Integration #
 ######################

-GITHUB_EXTENDED_PERMISSIONS = ['repo']
+GITHUB_EXTENDED_PERMISSIONS = ["repo"]

 #########################
 # Bitbucket Integration #
--- a/test.sh
+++ b/test.sh
@ -5,30 +5,91 @@ SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}"
 TEST_USER='test@example.com'
 TEST_PASS='test123TEST'
 COOKIE_FILE=$(mktemp)
-declare -a TEST_STRINGS=(
+
+# Courtesy of https://stackoverflow.com/a/2183063/90297
+trap_with_arg() {
+    func="$1" ; shift
+    for sig ; do
+        trap "$func $sig" "$sig"
+    done
+}
+
+DID_CLEAN_UP=0
+# the cleanup function will be the exit point
+cleanup () {
+  if [ "$DID_CLEAN_UP" -eq 1 ]; then
+    return 0;
+  fi
+  DID_CLEAN_UP=1
+
+  if [ "$1" != "EXIT" ]; then
+    echo "An error occurred, caught SIG$1";
+  fi
+
+  echo "Cleaning up..."
+  rm $COOKIE_FILE
+  echo "Done."
+}
+trap_with_arg cleanup ERR INT TERM EXIT
+
+get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; }
+sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; }
+
+login () {
+    INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective})
+    if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then
+        echo "Initial /auth/login/ redirect failed, exiting..."
+        echo "$INITIAL_AUTH_REDIRECT"
+        exit -1
+    fi
+
+    CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" '
+        /csrfmiddlewaretoken/ {
+        print $4 "=" $6;
+        exit;
+    }')
+
+    curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE";
+}
+
+LOGIN_RESPONSE=$(login);
+declare -a LOGIN_TEST_STRINGS=(
    '"isAuthenticated":true'
    '"username":"test@example.com"'
    '"isSuperuser":true'
 )
-
-INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective})
-if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then
-    echo "Initial /auth/login/ redirect failed, exiting..."
-    echo "$INITIAL_AUTH_REDIRECT"
-    exit -1
-fi
-
-CSRF_TOKEN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" '
-    /csrfmiddlewaretoken/ {
-      print $4 "=" $6;
-      exit;
-    }')
-LOGIN_RESPONSE=$(curl -sL -F 'op=login' -F "username=$TEST_USER" -F "password=$TEST_PASS" -F "$CSRF_TOKEN" "$SENTRY_TEST_HOST/auth/login/" -H "Referer: $SENTRY_TEST_HOST/auth/login/" -b "$COOKIE_FILE" -c "$COOKIE_FILE")
-
-TEST_RESULT=0
-for i in "${TEST_STRINGS[@]}"
+for i in "${LOGIN_TEST_STRINGS[@]}"
 do
   echo "Testing '$i'..."
   echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null
   echo "Pass."
 done
+
+# Set up initial/required settings (InstallWizard request)
+sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null
+
+SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }')
+
+TEST_EVENT_ID=$(docker run --rm --net host -e "SENTRY_DSN=$SENTRY_DSN" -v $(pwd):/work getsentry/sentry-cli send-event -m "a failure" -e task:create-user -e object:42 | tr -d '-')
+echo "Created event $TEST_EVENT_ID."
+
+EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/"
+export -f sentry_api_request get_csrf_token
+export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH
+printf "Checking its existence"
+timeout 15 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done'
+echo "";
+
+EVENT_RESPONSE=$(sentry_api_request "projects/sentry/internal/events/$TEST_EVENT_ID/")
+declare -a EVENT_TEST_STRINGS=(
+    '"eventID":"'"$TEST_EVENT_ID"'"'
+    '"message":"a failure"'
+    '"title":"a failure"'
+    '"object":"42"'
+)
+for i in "${EVENT_TEST_STRINGS[@]}"
+do
+   echo "Testing '$i'..."
+   echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null
+   echo "Pass."
+done