Merge pull request #13 from eidam/e/kv-gc

feat: move gc monitors from cron schedule to deploy postCommands

.github/workflows/deploy.yml

@@ -24,10 +24,12 @@ jobs:
         preCommands: |
           wrangler kv:namespace create KV_STATUS_PAGE
           apt-get update && apt-get install -y jq
-          kv_namespace_id=$(wrangler kv:namespace list | jq -c 'map(select(.title | contains("KV_STATUS_PAGE")))' | jq ".[0].id")
+          export KV_NAMESPACE_ID=$(wrangler kv:namespace list | jq -c 'map(select(.title | contains("KV_STATUS_PAGE")))' | jq -r ".[0].id")
           echo "[env.production]" >> wrangler.toml
-          echo "kv_namespaces = [{binding=\"KV_STATUS_PAGE\", id=${kv_namespace_id}}]" >> wrangler.toml
+          echo "kv_namespaces = [{binding=\"KV_STATUS_PAGE\", id=\"${KV_NAMESPACE_ID}\"}]" >> wrangler.toml
           [ -z "$SECRET_SLACK_WEBHOOK_URL" ] && echo "Secret SECRET_SLACK_WEBHOOK_URL not set, creating dummy one..." && SECRET_SLACK_WEBHOOK_URL="default-gh-action-secret" || true
+        postCommands: |
+          yarn kv-gc
         secrets: |
           SECRET_SLACK_WEBHOOK_URL
         environment: production

README.md

@@ -12,7 +12,7 @@ You'll need a [Cloudflare Workers account](https://dash.cloudflare.com/sign-up/w
 
 * A workers domain set up
 * The Workers Bundled subscription \($5/mo\)
-    * [Try it now with the free tier!](https://blog.cloudflare.com/workers-kv-free-tier/) Stay tuned while we make some changes so it will completely fit in the Free Tier with a 5min check interval.
+    * [Try it now with the free tier!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free. 
 * Some websites/APIs to watch 🙂
 
 Also, prepare the following secrets
@@ -40,7 +40,7 @@ You can either deploy with **Cloudflare Deploy Button** using GitHub Actions or
    - Value: your-slack-webhook-url
    ```
 3. Navigate to the **Actions** settings in your repository and enable them
-4. Edit [config.yaml](https://github.com/eidam/cf-workers-status-page/blob/main/config.yaml) to adjust configuration and list all of your websites/APIs you want to monitor
+4. Edit [config.yaml](./config.yaml) to adjust configuration and list all of your websites/APIs you want to monitor
 
    ```yaml
    settings:
@@ -74,16 +74,20 @@ You can either deploy with **Cloudflare Deploy Button** using GitHub Actions or
 6. 🎉
 7. _\(optional\)_ Go to [Cloudflare Workers settings](https://dash.cloudflare.com/?to=/workers) and assign custom domain/route
    * e.g. `status-page.eidam.dev/*` _\(make sure you include `/*` as the Worker also serve static files\)_
-8. _\(optional\)_ Edit [wrangler.toml](https://github.com/eidam/cf-workers-github-releases/blob/main/wrangler.toml) to adjust Worker settings or CRON Trigger schedule
+8. _\(optional\)_ Edit [wrangler.toml](./wrangler.toml) to adjust Worker settings or CRON Trigger schedule, especially if you are on [Workers Free plan](#workers-kv-free-tier)
 
 ### Deploy on your own
 
 You can clone the repository yourself and use Wrangler CLI to develop/deploy, extra list of things you need to take care of:
 
-* create KV namespace and add the `KV_STATUS_PAGE` binding to [wrangler.toml](https://github.com/eidam/cf-workers-github-releases/blob/main/wrangler.toml)
+* create KV namespace and add the `KV_STATUS_PAGE` binding to [wrangler.toml](./wrangler.toml)
 * create Worker secrets _\(optional\)_
   * `SECRET_SLACK_WEBHOOK_URL`
 
+## Workers KV free tier
+The Workers Free plan includes limited KV usage, in order to not deplete the quota and still have enough room for monitor status changes we recommend the following changes:
+* Change the CRON trigger to 5 minutes interval (`crons = ["*/5 * * * *"]`) in [wrangler.toml](./wrangler.toml)
+
 ## Known issues
 
 * **Max 25 monitors to watch in case you are using Slack notifications**, due to the limit of subrequests Cloudflare Worker can make \(50\).
@@ -93,3 +97,10 @@ You can clone the repository yourself and use Wrangler CLI to develop/deploy, ex
 * **KV replication lag** - You might get Slack notification instantly, however it may take couple of more seconds to see the change on your status page as [Cron Triggers are usually running on underutilized quiet hours machines](https://blog.cloudflare.com/introducing-cron-triggers-for-cloudflare-workers/#how-are-you-able-to-offer-this-feature-at-no-additional-cost).
 
 * **Initial delay (no data)** - It takes couple of minutes to schedule and run CRON Triggers for the first time
+
+* **Slack message for monitor just removed from the config** - It takes a couple of minutes to schedule new version of CRON Triggers, so older version of the configuration might be scheduled after deployment/gc - just ignore it or re-run the latest Github Action after a couple of minutes again.
+
+## Future plans
+Stay tuned for more features coming in, like leveraging the fact that CRON instances are scheduled around the world during the day
+ so we can monitor the response times. However, we will most probably wait for the [Durable Objects](https://blog.cloudflare.com/introducing-workers-durable-objects/) to be in open beta
+ as they are better fit to reliably store such info.

package.json

@@ -9,16 +9,18 @@
     "dev": "flareact dev",
     "build": "flareact build",
     "deploy": "flareact publish",
+    "kv-gc": "node ./src/cli/gcMonitors.js",
     "format": "prettier --write '**/*.{js,css,json,md}'"
   },
   "dependencies": {
-    "flareact": "^0.8.0",
+    "flareact": "^0.9.0",
     "laco": "^1.2.1",
     "laco-react": "^1.1.0",
     "react": "^16.13.1",
     "react-dom": "^16.13.1"
   },
   "devDependencies": {
+    "node-fetch": "^2.6.1",
     "prettier": "^1.18.2",
     "yaml-loader": "^0.6.0"
   }

src/cli/gcMonitors.js

@@ -0,0 +1,74 @@
+const yaml = require('yaml-loader')
+const fetch = require('node-fetch')
+const fs = require('fs')
+
+const accountId = process.env.CF_ACCOUNT_ID
+const namespaceId = process.env.KV_NAMESPACE_ID
+const apiToken = process.env.CF_API_TOKEN
+
+const kvPrefix = 's_'
+
+if (!accountId || !namespaceId || !apiToken) {
+  console.error("Missing required environment variables: CF_ACCOUNT_ID, KV_NAMESPACE_ID, CF_API_TOKEN")
+  process.exit(0)
+}
+
+async function getKvMonitors(kvPrefix) {
+  const init = {
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${apiToken}`,
+    },
+  }
+
+  const res = await fetch(
+    `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/keys?limit=100&prefix=${kvPrefix}`,
+    init,
+  )
+  const json = await res.json()
+  return json.result
+}
+
+async function deleteKvBulk(keys) {
+  const init = {
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${apiToken}`,
+    },
+    method: 'DELETE',
+    body: JSON.stringify(keys),
+  }
+
+  return await fetch(
+    `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/bulk`,
+    init,
+  )
+}
+
+function loadConfig() {
+  const configFile = fs.readFileSync('./config.yaml', 'utf8')
+  const config = yaml(configFile)
+  return JSON.parse(config)
+}
+
+getKvMonitors(kvPrefix).then(async kvMonitors => {
+  const config = loadConfig()
+  const monitors = config.monitors.map(key => {
+    return key.id
+  })
+  const kvState = kvMonitors.map(key => {
+    return key.name
+  })
+  const keysForRemoval = kvState.filter(
+    x => !monitors.includes(x.replace(kvPrefix, '')),
+  )
+
+  if (keysForRemoval.length > 0) {
+    console.log(
+      `Removing following keys from KV storage as they are no longer in the config: ${keysForRemoval.join(
+        ', ',
+      )}`,
+    )
+    await deleteKvBulk(keysForRemoval)
+  }
+}).catch(e => console.log(e))

src/functions/cronTrigger.js

@@ -3,7 +3,6 @@ import config from '../../config.yaml'
 import {
   setKV,
   getKVWithMetadata,
-  gcMonitors,
   getKV,
   notifySlack,
 } from './helpers'
@@ -70,8 +69,5 @@ export async function processCronTrigger(event) {
   const loc = res.headers.get('cf-ray').split('-')[1]
   await setKV('lastUpdate', Date.now(), { loc })
 
-  // gc monitor statuses
-  event.waitUntil(gcMonitors(config))
-
   return new Response('OK')
 }

src/functions/helpers.js

@@ -56,26 +56,6 @@ export async function deleteKV(key) {
   return KV_STATUS_PAGE.delete(key)
 }
 
-export async function gcMonitors(config) {
-  const checkKvPrefix = 's_'
-
-  const monitors = config.monitors.map(key => {
-    return key.id
-  })
-
-  const kvMonitors = await listKV(checkKvPrefix)
-  const kvState = kvMonitors.keys.map(key => {
-    return key.metadata.id
-  })
-
-  const keysForRemoval = kvState.filter(x => !monitors.includes(x))
-
-  for (const key of keysForRemoval) {
-    console.log('gc: deleting ' + checkKvPrefix + key)
-    await deleteKV(checkKvPrefix + key)
-  }
-}
-
 export async function notifySlack(monitor, newMetadata) {
   const payload = {
     attachments: [

yarn.lock

@@ -2947,10 +2947,10 @@ findup-sync@^3.0.0:
     micromatch "^3.0.4"
     resolve-dir "^1.0.1"
 
-flareact@^0.8.0:
+flareact@^0.9.0:
-  version "0.8.0"
+  version "0.9.0"
-  resolved "https://registry.yarnpkg.com/flareact/-/flareact-0.8.0.tgz#c7653f7278abee04353c5bf087af4105b7895c0a"
+  resolved "https://registry.yarnpkg.com/flareact/-/flareact-0.9.0.tgz#c16ded48f217010452a509e02b754f84eb26878c"
-  integrity sha512-ewjFqrxSXPBppyZtVBTD3W4iIcMB2wORpYX1QofOa9QRuy7dJ2nK4oMvCNmPJPelqdN5MF+fUBTIfMGqO3A1OA==
+  integrity sha512-YT1nGqusHTJDreU5gQezKQNU2Pszez+M3v5IrKIEtOD3ABQal+cVoWzRQGQTWMKryrUpWB0Z0nRhLYDutD8xdQ==
   dependencies:
     "@babel/core" "^7.11.0"
     "@babel/plugin-transform-runtime" "^7.11.0"
@@ -4354,6 +4354,11 @@ nice-try@^1.0.4:
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
   integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
 
+node-fetch@^2.6.1:
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
+  integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
+
 node-forge@^0.10.0:
   version "0.10.0"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3"