From c5b9232eb2786ba4023278f0605c1c44ce96e62e Mon Sep 17 00:00:00 2001 From: Adam Janis Date: Thu, 19 Nov 2020 18:43:33 +0100 Subject: [PATCH] feat: optimize KV storage read/write operations - the state is now stored in a single KV key - there is one write for cron and one read for render --- README.md | 6 +-- pages/index.js | 42 ++++----------- src/components/monitorHistogram.js | 10 ++-- src/components/monitorStatusHeader.js | 10 ++-- src/components/monitorStatusLabel.js | 6 +-- src/functions/cronTrigger.js | 73 +++++++++++++++------------ src/functions/helpers.js | 57 +++------------------ 7 files changed, 71 insertions(+), 133 deletions(-) diff --git a/README.md b/README.md index f5cbfc5..48fdbdc 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ You'll need a [Cloudflare Workers account](https://dash.cloudflare.com/sign-up/w * A workers domain set up * The Workers Bundled subscription \($5/mo\) - * [Try it now with the free tier!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free. + * [It works with Workers Free!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free. * Some websites/APIs to watch 🙂 Also, prepare the following secrets @@ -85,8 +85,8 @@ You can clone the repository yourself and use Wrangler CLI to develop/deploy, ex * `SECRET_SLACK_WEBHOOK_URL` ## Workers KV free tier -The Workers Free plan includes limited KV usage, in order to not deplete the quota and still have enough room for monitor status changes we recommend the following changes: -* Change the CRON trigger to 5 minutes interval (`crons = ["*/5 * * * *"]`) in [wrangler.toml](./wrangler.toml) +The Workers Free plan includes limited KV usage, but the quota is sufficient for 2-minute checks only +* Change the CRON trigger to 2 minutes interval (`crons = ["*/2 * * * *"]`) in [wrangler.toml](./wrangler.toml) ## Known issues diff --git a/pages/index.js b/pages/index.js index 85e364d..110d011 100644 --- a/pages/index.js +++ b/pages/index.js @@ -2,9 +2,7 @@ import Head from 'flareact/head' import MonitorHistogram from '../src/components/monitorHistogram' import { - getLastUpdate, getMonitors, - getMonitorsHistory, useKeyPress, } from '../src/functions/helpers' @@ -30,30 +28,13 @@ const filterByTerm = (term) => MonitorStore.set( export async function getEdgeProps() { // get KV data - const kvMonitors = await getMonitors() - const kvMonitorsFailedDays = await getMonitorsHistory() - const kvLastUpdate = await getLastUpdate() - - // prepare data maps for components - let monitorsOperational = true - let kvMonitorsMap = {} - kvMonitors.forEach(x => { - kvMonitorsMap[x.metadata.id] = x.metadata - if (x.metadata.operational === false) monitorsOperational = false - }) - - // transform KV list to array of failed days - const kvMonitorsFailedDaysArray = kvMonitorsFailedDays.map(x => { - return x.name - }) + const {value: kvMonitors, metadata: kvMonitorsMetadata } = await getMonitors() return { props: { config, - kvMonitorsMap, - kvMonitorsFailedDaysArray, - monitorsOperational, - kvLastUpdate, + kvMonitors: kvMonitors || {}, + kvMonitorsMetadata: kvMonitorsMetadata || {} }, // Revalidate these props once every x seconds revalidate: 5, @@ -62,10 +43,8 @@ export async function getEdgeProps() { export default function Index({ config, - kvMonitorsMap, - kvMonitorsFailedDaysArray, - monitorsOperational, - kvLastUpdate, + kvMonitors, + kvMonitorsMetadata, }) { const state = useStore(MonitorStore) const slash = useKeyPress('/') @@ -96,8 +75,7 @@ export default function Index({ /> {state.visible.map((monitor, key) => { return ( @@ -115,15 +93,13 @@ export default function Index({
{monitor.name}
diff --git a/src/components/monitorHistogram.js b/src/components/monitorHistogram.js index 7a0cd78..85f3178 100644 --- a/src/components/monitorHistogram.js +++ b/src/components/monitorHistogram.js @@ -1,8 +1,7 @@ import config from '../../config.yaml' export default function MonitorHistogram({ - kvMonitorsFailedDaysArray, - monitor, + monitorId, kvMonitor, }) { // create date and set date - daysInHistogram for the first day of the histogram @@ -12,20 +11,19 @@ export default function MonitorHistogram({ if (typeof window !== 'undefined') { return (
{Array.from(Array(config.settings.daysInHistogram).keys()).map(key => { date.setDate(date.getDate() + 1) const dayInHistogram = date.toISOString().split('T')[0] - const dayInHistogramKey = 'h_' + monitor.id + '_' + dayInHistogram let bg = '' let dayInHistogramLabel = config.settings.dayInHistogramNoData // filter all dates before first check, check the rest if (kvMonitor && kvMonitor.firstCheck <= dayInHistogram) { - if (!kvMonitorsFailedDaysArray.includes(dayInHistogramKey)) { + if (!kvMonitor.failedDays.includes(dayInHistogram)) { bg = 'green' dayInHistogramLabel = config.settings.dayInHistogramOperational } else { @@ -48,7 +46,7 @@ export default function MonitorHistogram({ } else { return (
Loading histogram ...
diff --git a/src/components/monitorStatusHeader.js b/src/components/monitorStatusHeader.js index b3b98d3..8cfd783 100644 --- a/src/components/monitorStatusHeader.js +++ b/src/components/monitorStatusHeader.js @@ -1,17 +1,15 @@ import config from '../../config.yaml' -export default function MonitorStatusHeader({ operational, lastUpdate }) { +export default function MonitorStatusHeader({kvMonitorsMetadata}) { let backgroundColor = 'green' let headerText = config.settings.allmonitorsOperational let textColor = 'black' - if (!operational) { + if (!kvMonitorsMetadata.monitorsOperational) { backgroundColor = 'yellow' headerText = config.settings.notAllmonitorsOperational } - const lastCheckAgo = Math.round((Date.now() - lastUpdate.value) / 1000) - return (
@@ -19,9 +17,9 @@ export default function MonitorStatusHeader({ operational, lastUpdate }) { {headerText}
{ - lastUpdate.metadata && typeof window !== 'undefined' && ( + kvMonitorsMetadata.lastUpdate && typeof window !== 'undefined' && (
- checked {lastCheckAgo} sec ago (from {lastUpdate.metadata.loc}) + checked {Math.round((Date.now() - kvMonitorsMetadata.lastUpdate.time) / 1000)} sec ago (from {kvMonitorsMetadata.lastUpdate.loc})
) } diff --git a/src/components/monitorStatusLabel.js b/src/components/monitorStatusLabel.js index 6a976eb..0c95175 100644 --- a/src/components/monitorStatusLabel.js +++ b/src/components/monitorStatusLabel.js @@ -1,11 +1,11 @@ import config from '../../config.yaml' -export default function MonitorStatusLabel({ kvMonitorsMap, monitor }) { +export default function MonitorStatusLabel({ kvMonitor }) { let labelColor = 'grey' let labelText = 'No data' - if (typeof kvMonitorsMap[monitor.id] !== 'undefined') { - if (kvMonitorsMap[monitor.id].operational) { + if (typeof kvMonitor !== 'undefined') { + if (kvMonitor.operational) { labelColor = 'green' labelText = config.settings.monitorLabelOperational } else { diff --git a/src/functions/cronTrigger.js b/src/functions/cronTrigger.js index 0aadc36..2676ac2 100644 --- a/src/functions/cronTrigger.js +++ b/src/functions/cronTrigger.js @@ -3,7 +3,6 @@ import config from '../../config.yaml' import { setKV, getKVWithMetadata, - getKV, notifySlack, } from './helpers' @@ -12,9 +11,29 @@ function getDate() { } export async function processCronTrigger(event) { + // Get monitors state from KV + let {value: monitorsState, metadata: monitorsStateMetadata} = await getKVWithMetadata('monitors_data', 'json') + + // Create empty state objects if not exists in KV storage yet + if (!monitorsState) { + monitorsState = {} + } + if (!monitorsStateMetadata) { + monitorsStateMetadata = {} + } + + // Reset default all monitors state to true + monitorsStateMetadata.monitorsOperational = true + for (const monitor of config.monitors) { + // Create default monitor state if does not exist yet + if (typeof monitorsState[monitor.id] === 'undefined') { + monitorsState[monitor.id] = {failedDays: []} + } + console.log(`Checking ${monitor.name} ...`) + // Fetch the monitors URL const init = { method: monitor.method || 'GET', redirect: monitor.followRedirect ? 'follow' : 'manual', @@ -24,50 +43,40 @@ export async function processCronTrigger(event) { } const checkResponse = await fetch(monitor.url, init) - const kvState = await getKVWithMetadata('s_' + monitor.id) + const monitorOperational = checkResponse.status === (monitor.expectStatus || 200) - // metadata from monitor settings - const newMetadata = { - operational: checkResponse.status === (monitor.expectStatus || 200), - id: monitor.id, - firstCheck: kvState.metadata ? kvState.metadata.firstCheck : getDate(), + // Send Slack message on monitor change + if (monitorsState[monitor.id].operational !== monitorOperational && typeof SECRET_SLACK_WEBHOOK_URL !== 'undefined' && SECRET_SLACK_WEBHOOK_URL !== 'default-gh-action-secret') { + event.waitUntil(notifySlack(monitor, monitorOperational)) } - // write current status if status changed or for first time - if ( - !kvState.metadata || - kvState.metadata.operational !== newMetadata.operational - ) { - console.log('Saving changed state..') + monitorsState[monitor.id].operational = checkResponse.status === (monitor.expectStatus || 200) + monitorsState[monitor.id].firstCheck = monitorsState[monitor.id].firstCheck || getDate() - // first try to notify Slack in case fetch() or other limit is reached - if (typeof SECRET_SLACK_WEBHOOK_URL !== 'undefined' && SECRET_SLACK_WEBHOOK_URL !== 'default-gh-action-secret') { - await notifySlack(monitor, newMetadata) - } + // Set monitorsOperational and push current day to failedDays + if (!monitorOperational) { + monitorsStateMetadata.monitorsOperational = false - await setKV('s_' + monitor.id, null, newMetadata) - } - - // write daily status if monitor is not operational - if (!newMetadata.operational) { - // try to get failed daily status first as KV read is cheaper than write - const kvFailedDayStatusKey = 'h_' + monitor.id + '_' + getDate() - const kvFailedDayStatus = await getKV(kvFailedDayStatusKey) - - // write if not found - if (!kvFailedDayStatus) { - console.log('Saving new failed daily status..') - await setKV(kvFailedDayStatusKey, null) + const failedDay = getDate() + if (!monitorsState[monitor.id].failedDays.includes(failedDay)) { + console.log('Saving new failed daily status ...') + monitorsState[monitor.id].failedDays.push(failedDay) } } } - // save last check timestamp including PoP location + // Get Worker PoP and save it to monitorsStateMetadata const res = await fetch('https://cloudflare-dns.com/dns-query', { method: 'OPTIONS', }) const loc = res.headers.get('cf-ray').split('-')[1] - await setKV('lastUpdate', Date.now(), { loc }) + monitorsStateMetadata.lastUpdate = { + loc, + time: Date.now() + } + + // Save monitorsState and monitorsStateMetadata to KV storage + await setKV('monitors_data', JSON.stringify(monitorsState), monitorsStateMetadata) return new Response('OK') } diff --git a/src/functions/helpers.js b/src/functions/helpers.js index 7df2a20..0853b1c 100644 --- a/src/functions/helpers.js +++ b/src/functions/helpers.js @@ -2,72 +2,29 @@ import config from '../../config.yaml' import {useEffect, useState} from 'react' export async function getMonitors() { - const monitors = await listKV('s_') - return monitors.keys -} - -export async function getMonitorsHistory() { - const monitorsHistory = await listKV('h_', 300) - return monitorsHistory.keys -} - -export async function getLastUpdate() { - return await getKVWithMetadata('lastUpdate') -} - -export async function listKV(prefix = '', cacheTtl = false) { - const cacheKey = 'list_' + prefix + '_' + process.env.BUILD_ID - - if (cacheTtl) { - const cachedResponse = await getKV(cacheKey) - if (cachedResponse) { - return JSON.parse(cachedResponse) - } - } - - let list = [] - let cursor = null - let res = {} - do { - res = await KV_STATUS_PAGE.list({ prefix: prefix, cursor }) - list = list.concat(res.keys) - cursor = res.cursor - } while (!res.list_complete) - - if (cacheTtl) { - await setKV(cacheKey, JSON.stringify({ keys: list }), null, cacheTtl) - } - return { keys: list } + return await getKVWithMetadata('monitors_data', "json") } export async function setKV(key, value, metadata, expirationTtl) { return KV_STATUS_PAGE.put(key, value, { metadata, expirationTtl }) } -export async function getKV(key, type = 'text') { - return KV_STATUS_PAGE.get(key, type) +export async function getKVWithMetadata(key, type = 'text') { + return KV_STATUS_PAGE.getWithMetadata(key, type) } -export async function getKVWithMetadata(key) { - return KV_STATUS_PAGE.getWithMetadata(key) -} - -export async function deleteKV(key) { - return KV_STATUS_PAGE.delete(key) -} - -export async function notifySlack(monitor, newMetadata) { +export async function notifySlack(monitor, operational) { const payload = { attachments: [ { - color: newMetadata.operational ? '#36a64f' : '#f2c744', + color: operational ? '#36a64f' : '#f2c744', blocks: [ { type: 'section', text: { type: 'mrkdwn', text: `Monitor *${monitor.name}* changed status to *${ - newMetadata.operational + operational ? config.settings.monitorLabelOperational : config.settings.monitorLabelNotOperational }*`, @@ -79,7 +36,7 @@ export async function notifySlack(monitor, newMetadata) { { type: 'mrkdwn', text: `${ - newMetadata.operational ? ':white_check_mark:' : ':x:' + operational ? ':white_check_mark:' : ':x:' } \`${monitor.method ? monitor.method : "GET"} ${monitor.url}\` - :eyes: <${ config.settings.url }|Status Page>`,