chore(perf): save only failed days to KV storage

config.yaml

@@ -1,7 +1,7 @@
 settings:
   title: "Status Page"
   logo: logo-192x192.png
-  daysInHistory: 90
+  daysInHistogram: 90
 
   allmonitorsOperational: "All Systems Operational"
   notAllmonitorsOperational: "Not All Systems Operational"

pages/index.js

@@ -13,7 +13,7 @@ import MonitorStatusLabel from '../src/components/monitorStatusLabel'
 export async function getEdgeProps() {
   // get KV data
   const kvMonitors = await getMonitors()
-  const kvMonitorsDays = await getMonitorsHistory()
+  const kvMonitorsFailedDays = await getMonitorsHistory()
   const kvLastUpdate = await getLastUpdate()
 
   // prepare data maps for components
@@ -24,16 +24,16 @@ export async function getEdgeProps() {
     if (x.metadata.operational === false) monitorsOperational = false
   })
 
-  let kvMonitorsDaysMap = {}
-  kvMonitorsDays.forEach(x => {
-    kvMonitorsDaysMap[x.name] = x.metadata.operational
+  // transform KV list to array of failed days
+  const kvMonitorsFailedDaysArray = kvMonitorsFailedDays.map(x => {
+    return x.name
   })
 
   return {
     props: {
       config,
       kvMonitorsMap,
-      kvMonitorsDaysMap,
+      kvMonitorsFailedDaysArray,
       monitorsOperational,
       kvLastUpdate,
     },
@@ -45,7 +45,7 @@ export async function getEdgeProps() {
 export default function Index({
   config,
   kvMonitorsMap,
-  kvMonitorsDaysMap,
+  kvMonitorsFailedDaysArray,
   monitorsOperational,
   kvLastUpdate,
 }) {
@@ -104,12 +104,13 @@ export default function Index({
               </div>
 
               <MonitorHistogram
-                kvMonitorsDaysMap={kvMonitorsDaysMap}
+                kvMonitorsFailedDaysArray={kvMonitorsFailedDaysArray}
                 monitor={monitor}
+                kvMonitor={kvMonitorsMap[monitor.id]}
               />
 
               <div className="horizontal flex between grey-text">
-                <div>{config.settings.daysInHistory} days ago</div>
+                <div>{config.settings.daysInHistogram} days ago</div>
                 <div>Today</div>
               </div>
             </div>

src/components/monitorHistogram.js

@@ -1,8 +1,13 @@
 import config from '../../config.yaml'
 
-export default function MonitorHistogram({ kvMonitorsDaysMap, monitor }) {
+export default function MonitorHistogram({
+  kvMonitorsFailedDaysArray,
+  monitor,
+  kvMonitor,
+}) {
+  // create date and set date - daysInHistogram for the first day of the histogram
   let date = new Date()
-  date.setDate(date.getDate() - config.settings.daysInHistory)
+  date.setDate(date.getDate() - config.settings.daysInHistogram)
 
   if (typeof window !== 'undefined') {
     return (
@@ -10,7 +15,7 @@ export default function MonitorHistogram({ kvMonitorsDaysMap, monitor }) {
         key={`${monitor.id}-histogram`}
         className="horizontal flex histogram"
       >
-        {Array.from(Array(config.settings.daysInHistory).keys()).map(key => {
+        {Array.from(Array(config.settings.daysInHistogram + 1).keys()).map(key => {
           date.setDate(date.getDate() + 1)
           const dayInHistory = date.toISOString().split('T')[0]
           const dayInHistoryKey = 'h_' + monitor.id + '_' + dayInHistory
@@ -18,11 +23,15 @@ export default function MonitorHistogram({ kvMonitorsDaysMap, monitor }) {
           let bg = ''
           let dayInHistoryStatus = 'No data'
 
-          if (typeof kvMonitorsDaysMap[dayInHistoryKey] !== 'undefined') {
-            bg = kvMonitorsDaysMap[dayInHistoryKey] ? 'green' : 'orange'
-            dayInHistoryStatus = kvMonitorsDaysMap[dayInHistoryKey]
-              ? 'No outages'
-              : 'Some outages'
+          // filter all dates before first check, check the rest
+          if (kvMonitor && kvMonitor.firstCheck <= dayInHistory) {
+            if (!kvMonitorsFailedDaysArray.includes(dayInHistoryKey)) {
+              bg = 'green'
+              dayInHistoryStatus = 'No outage recorded'
+            } else {
+              bg = 'orange'
+              dayInHistoryStatus = 'Some outages'
+            }
           }
 
           return (

src/functions/cronTrigger.js

@@ -1,6 +1,10 @@
 import config from '../../config.yaml'
 
-import { setKV, getKV, getKVWithMetadata, gcMonitors } from './helpers'
+import { setKV, getKVWithMetadata, gcMonitors, getKV } from './helpers'
+
+function getDate() {
+  return new Date().toISOString().split('T')[0]
+}
 
 export async function processCronTrigger(event) {
   for (const monitor of config.monitors) {
@@ -14,41 +18,43 @@ export async function processCronTrigger(event) {
       },
     }
 
-    const response = await fetch(monitor.url, init)
-    const monitorOperational = response.status === (monitor.expectStatus || 200)
-    const kvMonitor = await getKVWithMetadata('s_' + monitor.id)
+    const checkResponse = await fetch(monitor.url, init)
+    const kvState = await getKVWithMetadata('s_' + monitor.id)
 
     // metadata from monitor settings
-    const metadata = {
-      operational: monitorOperational,
-      statusCode: response.status,
+    const newMetadata = {
+      operational: checkResponse.status === (monitor.expectStatus || 200),
       id: monitor.id,
+      firstCheck: kvState.metadata ? kvState.metadata.firstCheck : getDate(),
     }
 
-    // write current status if status changed or for first time
+    // Write current status if status changed or for first time
     if (
-      !kvMonitor.metadata ||
-      kvMonitor.metadata.operational !== monitorOperational
+      !kvState.metadata ||
+      kvState.metadata.operational !== newMetadata.operational
     ) {
-      console.log('saving new results..')
+      console.log('Saving changed state..')
+
+      await setKV('s_' + monitor.id, null, newMetadata)
 
       if (typeof SECRET_SLACK_WEBHOOK !== 'undefined') {
-        await notifySlack(metadata)
+        await notifySlack(monitor, newMetadata)
       }
 
-      await setKV('s_' + monitor.id, null, metadata)
-    }
-
-    // check day status, write only on not operational or for first time
-    const kvDayStatusKey =
-      'h_' + monitor.id + '_' + new Date().toISOString().split('T')[0]
-    //console.log(kvDayStatusKey)
-    const kvDayStatus = await getKV(kvDayStatusKey)
-
-    if (!kvDayStatus || (kvDayStatus && !monitorOperational)) {
-      await setKV(kvDayStatusKey, null, metadata)
+      if (!newMetadata.operational) {
+        // try to get failed daily status first as KV read is cheaper than write
+        const kvFailedDayStatusKey = 'h_' + monitor.id + '_' + getDate()
+        const kvFailedDayStatus = await getKV(kvFailedDayStatusKey)
+
+        // write if not found
+        if (!kvFailedDayStatus) {
+          console.log('Saving new failed daily status..')
+          await setKV(kvFailedDayStatusKey, null)
+        }
+      }
     }
 
+    // save last check timestamp
     await setKV('lastUpdate', Date.now())
   }
   await gcMonitors(config)

src/functions/helpers.js

@@ -4,7 +4,7 @@ export async function getMonitors() {
 }
 
 export async function getMonitorsHistory() {
-  const monitorsHistory = await listKV('h_', 600)
+  const monitorsHistory = await listKV('h_', 300)
   return monitorsHistory.keys
 }
 
@@ -14,10 +14,12 @@ export async function getLastUpdate() {
 
 export async function listKV(prefix = '', cacheTtl = false) {
   const cacheKey = 'list_' + prefix + '_' + process.env.BUILD_ID
-  const cachedResponse = await getKV(cacheKey)
 
-  if (cacheTtl && cachedResponse) {
-    return JSON.parse(cachedResponse)
+  if (cacheTtl) {
+    const cachedResponse = await getKV(cacheKey)
+    if (cachedResponse) {
+      return JSON.parse(cachedResponse)
+    }
   }
 
   let list = []
@@ -30,7 +32,7 @@ export async function listKV(prefix = '', cacheTtl = false) {
   } while (!res.list_complete)
 
   if (cacheTtl) {
-    await setKV(cacheKey, JSON.stringify({ keys: list }), null, 600)
+    await setKV(cacheKey, JSON.stringify({ keys: list }), null, cacheTtl)
   }
   return { keys: list }
 }