kmlabz/sentry-onpremise
kmlabz/sentry-onpremise
Archived
2
0
Fork 0
Code Pull Requests Releases Activity

feat(upgrade): Add automatic Snuba migration for upgrades (#292)

Browse Source 
Upgrades existing events from last `$SENTRY_EVENT_RETENTION_DAYS`
to Snuba automatically. Relies on getsentry/sentry#15934.
This commit is contained in:
Burak Yigit Kaya 2019-12-09 21:33:02 +03:00 committed by GitHub
parent 3c9e8c28ff
commit ba67058070
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 48 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
install.sh
View File

@ -16,7 +16,7 @@ cleanup () {
return 0;
fi
echo "Cleaning up..."
docker-compose down &> /dev/null
docker-compose stop &> /dev/null
DID_CLEAN_UP=1
}
trap cleanup ERR INT TERM
@ -55,6 +55,40 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then
exit -1
fi
# Ensure nothing is working while we install/update
docker-compose stop
echo ""
echo "Creating volumes for persistent storage..."
echo "Created $(docker volume create --name=sentry-data)."
echo "Created $(docker volume create --name=sentry-postgres)."
echo "Created $(docker volume create --name=sentry-redis)."
echo "Created $(docker volume create --name=sentry-zookeeper)."
echo "Created $(docker volume create --name=sentry-kafka)."
echo "Created $(docker volume create --name=sentry-clickhouse)."
echo "Created $(docker volume create --name=sentry-symbolicator)."
echo ""
ensure_file_from_example $SENTRY_CONFIG_PY
ensure_file_from_example $SENTRY_CONFIG_YML
ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS
echo ""
echo "Generating secret key..."
# This is to escape the secret key to be used in sed below
SECRET_KEY=$(head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g')
sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML
echo "Secret key written to $SENTRY_CONFIG_YML"
echo ""
echo "Building and tagging Docker images..."
echo ""
# Build the sentry onpremise image first as it is needed for the cron image
docker-compose build --force-rm web
docker-compose build --force-rm
echo ""
echo "Docker images built."
# Very naively check whether there's an existing sentry-postgres volume and the PG version in it
if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then
# If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume
@ -75,37 +109,6 @@ if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm
docker volume rm sentry-postgres-new
fi
echo ""
ensure_file_from_example $SENTRY_CONFIG_PY
ensure_file_from_example $SENTRY_CONFIG_YML
ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS
echo ""
echo "Creating volumes for persistent storage..."
echo "Created $(docker volume create --name=sentry-data)."
echo "Created $(docker volume create --name=sentry-postgres)."
echo "Created $(docker volume create --name=sentry-redis)."
echo "Created $(docker volume create --name=sentry-zookeeper)."
echo "Created $(docker volume create --name=sentry-kafka)."
echo "Created $(docker volume create --name=sentry-clickhouse)."
echo "Created $(docker volume create --name=sentry-symbolicator)."
echo ""
echo "Generating secret key..."
# This is to escape the secret key to be used in sed below
SECRET_KEY=$(head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g')
sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML
echo "Secret key written to $SENTRY_CONFIG_YML"
echo ""
echo "Building and tagging Docker images..."
echo ""
# Build the sentry onpremise image first as it is needed for the cron image
docker-compose build --force-rm web
docker-compose build --force-rm
echo ""
echo "Docker images built."
echo ""
echo "Setting up database..."
if [ $CI ]; then
@ -120,6 +123,13 @@ else
docker-compose run --rm web upgrade
fi
SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l")
if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then
echo "Migrating file storage..."
docker run --rm -v sentry-data:/data alpine ash -c \
"mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files"
fi
echo "Boostrapping Snuba..."
docker-compose up -d kafka redis clickhouse
until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do
@ -130,12 +140,12 @@ until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --que
done;
echo ""
SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l")
if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then
echo "Migrating file storage..."
docker run --rm -v sentry-data:/data alpine ash -c \
"mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files"
fi
set -o allexport
source .env
set +o allexport
echo "Migrating old events for the last $SENTRY_EVENT_RETENTION_DAYS days..."
docker-compose run --rm web django backfill_eventstream --no-input --last-days $SENTRY_EVENT_RETENTION_DAYS
echo ""
cleanup