Merge pull request #14 from eidam/e/kv-resources-optimization

feat: optimize KV storage read/write operations

README.md

@@ -12,7 +12,7 @@ You'll need a [Cloudflare Workers account](https://dash.cloudflare.com/sign-up/w
 
 * A workers domain set up
 * The Workers Bundled subscription \($5/mo\)
-    * [Try it now with the free tier!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free. 
+    * [It works with Workers Free!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free. 
 * Some websites/APIs to watch 🙂
 
 Also, prepare the following secrets
@@ -85,8 +85,8 @@ You can clone the repository yourself and use Wrangler CLI to develop/deploy, ex
   * `SECRET_SLACK_WEBHOOK_URL`
 
 ## Workers KV free tier
-The Workers Free plan includes limited KV usage, in order to not deplete the quota and still have enough room for monitor status changes we recommend the following changes:
-* Change the CRON trigger to 5 minutes interval (`crons = ["*/5 * * * *"]`) in [wrangler.toml](./wrangler.toml)
+The Workers Free plan includes limited KV usage, but the quota is sufficient for 2-minute checks only 
+* Change the CRON trigger to 2 minutes interval (`crons = ["*/2 * * * *"]`) in [wrangler.toml](./wrangler.toml)
 
 ## Known issues
 

pages/index.js

@@ -2,9 +2,7 @@ import Head from 'flareact/head'
 import MonitorHistogram from '../src/components/monitorHistogram'
 
 import {
-  getLastUpdate,
   getMonitors,
-  getMonitorsHistory,
   useKeyPress,
 } from '../src/functions/helpers'
 
@@ -30,30 +28,13 @@ const filterByTerm = (term) => MonitorStore.set(
 
 export async function getEdgeProps() {
   // get KV data
-  const kvMonitors = await getMonitors()
-  const kvMonitorsFailedDays = await getMonitorsHistory()
-  const kvLastUpdate = await getLastUpdate()
-
-  // prepare data maps for components
-  let monitorsOperational = true
-  let kvMonitorsMap = {}
-  kvMonitors.forEach(x => {
-    kvMonitorsMap[x.metadata.id] = x.metadata
-    if (x.metadata.operational === false) monitorsOperational = false
-  })
-
-  // transform KV list to array of failed days
-  const kvMonitorsFailedDaysArray = kvMonitorsFailedDays.map(x => {
-    return x.name
-  })
+  const {value: kvMonitors, metadata: kvMonitorsMetadata } = await getMonitors()
 
   return {
     props: {
       config,
-      kvMonitorsMap,
-      kvMonitorsFailedDaysArray,
-      monitorsOperational,
-      kvLastUpdate,
+      kvMonitors: kvMonitors || {},
+      kvMonitorsMetadata: kvMonitorsMetadata || {}
     },
     // Revalidate these props once every x seconds
     revalidate: 5,
@@ -62,10 +43,8 @@ export async function getEdgeProps() {
 
 export default function Index({
   config,
-  kvMonitorsMap,
-  kvMonitorsFailedDaysArray,
-  monitorsOperational,
-  kvLastUpdate,
+  kvMonitors,
+  kvMonitorsMetadata,
 }) {
   const state = useStore(MonitorStore)
   const slash = useKeyPress('/')
@@ -96,8 +75,7 @@ export default function Index({
           />
         </div>
         <MonitorStatusHeader
-          operational={monitorsOperational}
-          lastUpdate={kvLastUpdate}
+          kvMonitorsMetadata={kvMonitorsMetadata}
         />
         {state.visible.map((monitor, key) => {
           return (
@@ -115,15 +93,13 @@ export default function Index({
                   <div className="content">{monitor.name}</div>
                 </div>
                 <MonitorStatusLabel
-                  kvMonitorsMap={kvMonitorsMap}
-                  monitor={monitor}
+                  kvMonitor={kvMonitors[monitor.id]}
                 />
               </div>
 
               <MonitorHistogram
-                kvMonitorsFailedDaysArray={kvMonitorsFailedDaysArray}
-                monitor={monitor}
-                kvMonitor={kvMonitorsMap[monitor.id]}
+                monitorId={monitor.id}
+                kvMonitor={kvMonitors[monitor.id]}
               />
 
               <div className="horizontal flex between grey-text">

src/components/monitorHistogram.js

@@ -1,8 +1,7 @@
 import config from '../../config.yaml'
 
 export default function MonitorHistogram({
-  kvMonitorsFailedDaysArray,
-  monitor,
+  monitorId,
   kvMonitor,
 }) {
   // create date and set date - daysInHistogram for the first day of the histogram
@@ -12,20 +11,19 @@ export default function MonitorHistogram({
   if (typeof window !== 'undefined') {
     return (
       <div
-        key={`${monitor.id}-histogram`}
+        key={`${monitorId}-histogram`}
         className="horizontal flex histogram"
       >
         {Array.from(Array(config.settings.daysInHistogram).keys()).map(key => {
           date.setDate(date.getDate() + 1)
           const dayInHistogram = date.toISOString().split('T')[0]
-          const dayInHistogramKey = 'h_' + monitor.id + '_' + dayInHistogram
 
           let bg = ''
           let dayInHistogramLabel = config.settings.dayInHistogramNoData
 
           // filter all dates before first check, check the rest
           if (kvMonitor && kvMonitor.firstCheck <= dayInHistogram) {
-            if (!kvMonitorsFailedDaysArray.includes(dayInHistogramKey)) {
+            if (!kvMonitor.failedDays.includes(dayInHistogram)) {
               bg = 'green'
               dayInHistogramLabel = config.settings.dayInHistogramOperational
             } else {
@@ -48,7 +46,7 @@ export default function MonitorHistogram({
   } else {
     return (
       <div
-        key={`${monitor.id}-histogram`}
+        key={`${monitorId}-histogram`}
         className="horizontal flex histogram"
       >
         <div className="grey-text">Loading histogram ...</div>

src/components/monitorStatusHeader.js

@@ -1,17 +1,15 @@
 import config from '../../config.yaml'
 
-export default function MonitorStatusHeader({ operational, lastUpdate }) {
+export default function MonitorStatusHeader({kvMonitorsMetadata}) {
   let backgroundColor = 'green'
   let headerText = config.settings.allmonitorsOperational
   let textColor = 'black'
 
-  if (!operational) {
+  if (!kvMonitorsMetadata.monitorsOperational) {
     backgroundColor = 'yellow'
     headerText = config.settings.notAllmonitorsOperational
   }
 
-  const lastCheckAgo = Math.round((Date.now() - lastUpdate.value) / 1000)
-
   return (
     <div className={`ui inverted segment ${backgroundColor}`}>
       <div className="horizontal flex between">
@@ -19,9 +17,9 @@ export default function MonitorStatusHeader({ operational, lastUpdate }) {
           {headerText}
         </div>
         {
-          lastUpdate.metadata && typeof window !== 'undefined' && (
+          kvMonitorsMetadata.lastUpdate && typeof window !== 'undefined' && (
           <div className={`${textColor}-text`}>
-            checked {lastCheckAgo} sec ago (from {lastUpdate.metadata.loc})
+            checked {Math.round((Date.now() - kvMonitorsMetadata.lastUpdate.time) / 1000)} sec ago (from {kvMonitorsMetadata.lastUpdate.loc})
           </div>
           )
         }

src/components/monitorStatusLabel.js

@@ -1,11 +1,11 @@
 import config from '../../config.yaml'
 
-export default function MonitorStatusLabel({ kvMonitorsMap, monitor }) {
+export default function MonitorStatusLabel({ kvMonitor }) {
   let labelColor = 'grey'
   let labelText = 'No data'
 
-  if (typeof kvMonitorsMap[monitor.id] !== 'undefined') {
-    if (kvMonitorsMap[monitor.id].operational) {
+  if (typeof kvMonitor !== 'undefined') {
+    if (kvMonitor.operational) {
       labelColor = 'green'
       labelText = config.settings.monitorLabelOperational
     } else {

src/functions/cronTrigger.js

@@ -3,7 +3,6 @@ import config from '../../config.yaml'
 import {
   setKV,
   getKVWithMetadata,
-  getKV,
   notifySlack,
 } from './helpers'
 
@@ -12,9 +11,29 @@ function getDate() {
 }
 
 export async function processCronTrigger(event) {
+  // Get monitors state from KV
+  let {value: monitorsState, metadata: monitorsStateMetadata} = await getKVWithMetadata('monitors_data', 'json')
+
+  // Create empty state objects if not exists in KV storage yet
+  if (!monitorsState) {
+    monitorsState = {}
+  }
+  if (!monitorsStateMetadata) {
+    monitorsStateMetadata = {}
+  }
+
+  // Reset default all monitors state to true
+  monitorsStateMetadata.monitorsOperational = true
+
   for (const monitor of config.monitors) {
+    // Create default monitor state if does not exist yet
+    if (typeof monitorsState[monitor.id] === 'undefined') {
+      monitorsState[monitor.id] = {failedDays: []}
+    }
+
     console.log(`Checking ${monitor.name} ...`)
 
+    // Fetch the monitors URL
     const init = {
       method: monitor.method || 'GET',
       redirect: monitor.followRedirect ? 'follow' : 'manual',
@@ -24,50 +43,40 @@ export async function processCronTrigger(event) {
     }
 
     const checkResponse = await fetch(monitor.url, init)
-    const kvState = await getKVWithMetadata('s_' + monitor.id)
+    const monitorOperational = checkResponse.status === (monitor.expectStatus || 200)
 
-    // metadata from monitor settings
-    const newMetadata = {
-      operational: checkResponse.status === (monitor.expectStatus || 200),
-      id: monitor.id,
-      firstCheck: kvState.metadata ? kvState.metadata.firstCheck : getDate(),
+    // Send Slack message on monitor change
+    if (monitorsState[monitor.id].operational !== monitorOperational && typeof SECRET_SLACK_WEBHOOK_URL !== 'undefined' && SECRET_SLACK_WEBHOOK_URL !== 'default-gh-action-secret') {
+        event.waitUntil(notifySlack(monitor, monitorOperational))
     }
 
-    // write current status if status changed or for first time
-    if (
-      !kvState.metadata ||
-      kvState.metadata.operational !== newMetadata.operational
-    ) {
-      console.log('Saving changed state..')
+    monitorsState[monitor.id].operational = checkResponse.status === (monitor.expectStatus || 200)
+    monitorsState[monitor.id].firstCheck = monitorsState[monitor.id].firstCheck || getDate()
 
-      // first try to notify Slack in case fetch() or other limit is reached
-      if (typeof SECRET_SLACK_WEBHOOK_URL !== 'undefined' && SECRET_SLACK_WEBHOOK_URL !== 'default-gh-action-secret') {
-        await notifySlack(monitor, newMetadata)
-      }
+    // Set monitorsOperational and push current day to failedDays
+    if (!monitorOperational) {
+      monitorsStateMetadata.monitorsOperational = false
 
-      await setKV('s_' + monitor.id, null, newMetadata)
-    }
-
-    // write daily status if monitor is not operational
-    if (!newMetadata.operational) {
-      // try to get failed daily status first as KV read is cheaper than write
-      const kvFailedDayStatusKey = 'h_' + monitor.id + '_' + getDate()
-      const kvFailedDayStatus = await getKV(kvFailedDayStatusKey)
-
-      // write if not found
-      if (!kvFailedDayStatus) {
-        console.log('Saving new failed daily status..')
-        await setKV(kvFailedDayStatusKey, null)
+      const failedDay = getDate()
+      if (!monitorsState[monitor.id].failedDays.includes(failedDay)) {
+        console.log('Saving new failed daily status ...')
+        monitorsState[monitor.id].failedDays.push(failedDay)
       }
     }
   }
 
-  // save last check timestamp including PoP location
+  // Get Worker PoP and save it to monitorsStateMetadata
   const res = await fetch('https://cloudflare-dns.com/dns-query', {
     method: 'OPTIONS',
   })
   const loc = res.headers.get('cf-ray').split('-')[1]
-  await setKV('lastUpdate', Date.now(), { loc })
+  monitorsStateMetadata.lastUpdate = {
+    loc,
+    time: Date.now()
+  }
+
+  // Save monitorsState and monitorsStateMetadata to KV storage
+  await setKV('monitors_data', JSON.stringify(monitorsState), monitorsStateMetadata)
 
   return new Response('OK')
 }

src/functions/helpers.js

@@ -2,72 +2,29 @@ import config from '../../config.yaml'
 import {useEffect, useState} from 'react'
 
 export async function getMonitors() {
-  const monitors = await listKV('s_')
-  return monitors.keys
-}
-
-export async function getMonitorsHistory() {
-  const monitorsHistory = await listKV('h_', 300)
-  return monitorsHistory.keys
-}
-
-export async function getLastUpdate() {
-  return await getKVWithMetadata('lastUpdate')
-}
-
-export async function listKV(prefix = '', cacheTtl = false) {
-  const cacheKey = 'list_' + prefix + '_' + process.env.BUILD_ID
-
-  if (cacheTtl) {
-    const cachedResponse = await getKV(cacheKey)
-    if (cachedResponse) {
-      return JSON.parse(cachedResponse)
-    }
-  }
-
-  let list = []
-  let cursor = null
-  let res = {}
-  do {
-    res = await KV_STATUS_PAGE.list({ prefix: prefix, cursor })
-    list = list.concat(res.keys)
-    cursor = res.cursor
-  } while (!res.list_complete)
-
-  if (cacheTtl) {
-    await setKV(cacheKey, JSON.stringify({ keys: list }), null, cacheTtl)
-  }
-  return { keys: list }
+  return await getKVWithMetadata('monitors_data', "json")
 }
 
 export async function setKV(key, value, metadata, expirationTtl) {
   return KV_STATUS_PAGE.put(key, value, { metadata, expirationTtl })
 }
 
-export async function getKV(key, type = 'text') {
-  return KV_STATUS_PAGE.get(key, type)
+export async function getKVWithMetadata(key, type = 'text') {
+  return KV_STATUS_PAGE.getWithMetadata(key, type)
 }
 
-export async function getKVWithMetadata(key) {
-  return KV_STATUS_PAGE.getWithMetadata(key)
-}
-
-export async function deleteKV(key) {
-  return KV_STATUS_PAGE.delete(key)
-}
-
-export async function notifySlack(monitor, newMetadata) {
+export async function notifySlack(monitor, operational) {
   const payload = {
     attachments: [
       {
-        color: newMetadata.operational ? '#36a64f' : '#f2c744',
+        color: operational ? '#36a64f' : '#f2c744',
         blocks: [
           {
             type: 'section',
             text: {
               type: 'mrkdwn',
               text: `Monitor *${monitor.name}* changed status to *${
-                newMetadata.operational
+                operational
                   ? config.settings.monitorLabelOperational
                   : config.settings.monitorLabelNotOperational
               }*`,
@@ -79,7 +36,7 @@ export async function notifySlack(monitor, newMetadata) {
               {
                 type: 'mrkdwn',
                 text: `${
-                  newMetadata.operational ? ':white_check_mark:' : ':x:'
+                  operational ? ':white_check_mark:' : ':x:'
                 } \`${monitor.method ? monitor.method : "GET"} ${monitor.url}\` - :eyes: <${
                   config.settings.url
                 }|Status Page>`,