tormakris
/
cf-workers-status-page
mirror of https://github.com/tormachris/cf-workers-status-page.git
1
0
Fork 0
Code Releases Activity

Merge pull request #14 from eidam/e/kv-resources-optimization 

feat: optimize KV storage read/write operations
This commit is contained in:
Adam Janiš 2020-11-19 20:35:28 +01:00 committed by GitHub
parent 293dff9425 c5b9232eb2
commit 42f422c455
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 71 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -12,7 +12,7 @@ You'll need a [Cloudflare Workers account](https://dash.cloudflare.com/sign-up/w
* A workers domain set up
* The Workers Bundled subscription \($5/mo\)
* [Try it now with the free tier!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free.
* [It works with Workers Free!](https://blog.cloudflare.com/workers-kv-free-tier/) Check [more info](#workers-kv-free-tier) on how to run on Workers Free.
* Some websites/APIs to watch 🙂
Also, prepare the following secrets
@ -85,8 +85,8 @@ You can clone the repository yourself and use Wrangler CLI to develop/deploy, ex
* `SECRET_SLACK_WEBHOOK_URL`
## Workers KV free tier
The Workers Free plan includes limited KV usage, in order to not deplete the quota and still have enough room for monitor status changes we recommend the following changes:
* Change the CRON trigger to 5 minutes interval (`crons = ["*/5 * * * *"]`) in [wrangler.toml](./wrangler.toml)
The Workers Free plan includes limited KV usage, but the quota is sufficient for 2-minute checks only
* Change the CRON trigger to 2 minutes interval (`crons = ["*/2 * * * *"]`) in [wrangler.toml](./wrangler.toml)
## Known issues

42
pages/index.js
View File

@ -2,9 +2,7 @@ import Head from 'flareact/head'
import MonitorHistogram from '../src/components/monitorHistogram'
import {
getLastUpdate,
getMonitors,
getMonitorsHistory,
useKeyPress,
} from '../src/functions/helpers'
@ -30,30 +28,13 @@ const filterByTerm = (term) => MonitorStore.set(
export async function getEdgeProps() {
// get KV data
const kvMonitors = await getMonitors()
const kvMonitorsFailedDays = await getMonitorsHistory()
const kvLastUpdate = await getLastUpdate()
// prepare data maps for components
let monitorsOperational = true
let kvMonitorsMap = {}
kvMonitors.forEach(x => {
kvMonitorsMap[x.metadata.id] = x.metadata
if (x.metadata.operational === false) monitorsOperational = false
})
// transform KV list to array of failed days
const kvMonitorsFailedDaysArray = kvMonitorsFailedDays.map(x => {
return x.name
})
const {value: kvMonitors, metadata: kvMonitorsMetadata } = await getMonitors()
return {
props: {
config,
kvMonitorsMap,
kvMonitorsFailedDaysArray,
monitorsOperational,
kvLastUpdate,
kvMonitors: kvMonitors || {},
kvMonitorsMetadata: kvMonitorsMetadata || {}
},
// Revalidate these props once every x seconds
revalidate: 5,
@ -62,10 +43,8 @@ export async function getEdgeProps() {
export default function Index({
config,
kvMonitorsMap,
kvMonitorsFailedDaysArray,
monitorsOperational,
kvLastUpdate,
kvMonitors,
kvMonitorsMetadata,
}) {
const state = useStore(MonitorStore)
const slash = useKeyPress('/')
@ -96,8 +75,7 @@ export default function Index({
/>
</div>
<MonitorStatusHeader
operational={monitorsOperational}
lastUpdate={kvLastUpdate}
kvMonitorsMetadata={kvMonitorsMetadata}
/>
{state.visible.map((monitor, key) => {
return (
@ -115,15 +93,13 @@ export default function Index({
<div className="content">{monitor.name}</div>
</div>
<MonitorStatusLabel
kvMonitorsMap={kvMonitorsMap}
monitor={monitor}
kvMonitor={kvMonitors[monitor.id]}
/>
</div>
<MonitorHistogram
kvMonitorsFailedDaysArray={kvMonitorsFailedDaysArray}
monitor={monitor}
kvMonitor={kvMonitorsMap[monitor.id]}
monitorId={monitor.id}
kvMonitor={kvMonitors[monitor.id]}
/>
<div className="horizontal flex between grey-text">

10
src/components/monitorHistogram.js
View File

@ -1,8 +1,7 @@
import config from '../../config.yaml'
export default function MonitorHistogram({
kvMonitorsFailedDaysArray,
monitor,
monitorId,
kvMonitor,
}) {
// create date and set date - daysInHistogram for the first day of the histogram
@ -12,20 +11,19 @@ export default function MonitorHistogram({
if (typeof window !== 'undefined') {
return (
<div
key={`${monitor.id}-histogram`}
key={`${monitorId}-histogram`}
className="horizontal flex histogram"
>
{Array.from(Array(config.settings.daysInHistogram).keys()).map(key => {
date.setDate(date.getDate() + 1)
const dayInHistogram = date.toISOString().split('T')[0]
const dayInHistogramKey = 'h_' + monitor.id + '_' + dayInHistogram
let bg = ''
let dayInHistogramLabel = config.settings.dayInHistogramNoData
// filter all dates before first check, check the rest
if (kvMonitor && kvMonitor.firstCheck <= dayInHistogram) {
if (!kvMonitorsFailedDaysArray.includes(dayInHistogramKey)) {
if (!kvMonitor.failedDays.includes(dayInHistogram)) {
bg = 'green'
dayInHistogramLabel = config.settings.dayInHistogramOperational
} else {
@ -48,7 +46,7 @@ export default function MonitorHistogram({
} else {
return (
<div
key={`${monitor.id}-histogram`}
key={`${monitorId}-histogram`}
className="horizontal flex histogram"
>
<div className="grey-text">Loading histogram ...</div>

10
src/components/monitorStatusHeader.js
View File

@ -1,17 +1,15 @@
import config from '../../config.yaml'
export default function MonitorStatusHeader({ operational, lastUpdate }) {
export default function MonitorStatusHeader({kvMonitorsMetadata}) {
let backgroundColor = 'green'
let headerText = config.settings.allmonitorsOperational
let textColor = 'black'
if (!operational) {
if (!kvMonitorsMetadata.monitorsOperational) {
backgroundColor = 'yellow'
headerText = config.settings.notAllmonitorsOperational
}
const lastCheckAgo = Math.round((Date.now() - lastUpdate.value) / 1000)
return (
<div className={`ui inverted segment ${backgroundColor}`}>
<div className="horizontal flex between">
@ -19,9 +17,9 @@ export default function MonitorStatusHeader({ operational, lastUpdate }) {
{headerText}
</div>
{
lastUpdate.metadata && typeof window !== 'undefined' && (
kvMonitorsMetadata.lastUpdate && typeof window !== 'undefined' && (
<div className={`${textColor}-text`}>
checked {lastCheckAgo} sec ago (from {lastUpdate.metadata.loc})
checked {Math.round((Date.now() - kvMonitorsMetadata.lastUpdate.time) / 1000)} sec ago (from {kvMonitorsMetadata.lastUpdate.loc})
</div>
)
}

6
src/components/monitorStatusLabel.js
View File

@ -1,11 +1,11 @@
import config from '../../config.yaml'
export default function MonitorStatusLabel({ kvMonitorsMap, monitor }) {
export default function MonitorStatusLabel({ kvMonitor }) {
let labelColor = 'grey'
let labelText = 'No data'
if (typeof kvMonitorsMap[monitor.id] !== 'undefined') {
if (kvMonitorsMap[monitor.id].operational) {
if (typeof kvMonitor !== 'undefined') {
if (kvMonitor.operational) {
labelColor = 'green'
labelText = config.settings.monitorLabelOperational
} else {

73
src/functions/cronTrigger.js
View File

@ -3,7 +3,6 @@ import config from '../../config.yaml'
import {
setKV,
getKVWithMetadata,
getKV,
notifySlack,
} from './helpers'
@ -12,9 +11,29 @@ function getDate() {
}
export async function processCronTrigger(event) {
// Get monitors state from KV
let {value: monitorsState, metadata: monitorsStateMetadata} = await getKVWithMetadata('monitors_data', 'json')
// Create empty state objects if not exists in KV storage yet
if (!monitorsState) {
monitorsState = {}
}
if (!monitorsStateMetadata) {
monitorsStateMetadata = {}
}
// Reset default all monitors state to true
monitorsStateMetadata.monitorsOperational = true
for (const monitor of config.monitors) {
// Create default monitor state if does not exist yet
if (typeof monitorsState[monitor.id] === 'undefined') {
monitorsState[monitor.id] = {failedDays: []}
}
console.log(`Checking ${monitor.name} ...`)
// Fetch the monitors URL
const init = {
method: monitor.method || 'GET',
redirect: monitor.followRedirect ? 'follow' : 'manual',
@ -24,50 +43,40 @@ export async function processCronTrigger(event) {
}
const checkResponse = await fetch(monitor.url, init)
const kvState = await getKVWithMetadata('s_' + monitor.id)
const monitorOperational = checkResponse.status === (monitor.expectStatus || 200)
// metadata from monitor settings
const newMetadata = {
operational: checkResponse.status === (monitor.expectStatus || 200),
id: monitor.id,
firstCheck: kvState.metadata ? kvState.metadata.firstCheck : getDate(),
// Send Slack message on monitor change
if (monitorsState[monitor.id].operational !== monitorOperational && typeof SECRET_SLACK_WEBHOOK_URL !== 'undefined' && SECRET_SLACK_WEBHOOK_URL !== 'default-gh-action-secret') {
event.waitUntil(notifySlack(monitor, monitorOperational))
}
// write current status if status changed or for first time
if (
!kvState.metadata ||
kvState.metadata.operational !== newMetadata.operational
) {
console.log('Saving changed state..')
monitorsState[monitor.id].operational = checkResponse.status === (monitor.expectStatus || 200)
monitorsState[monitor.id].firstCheck = monitorsState[monitor.id].firstCheck || getDate()
// first try to notify Slack in case fetch() or other limit is reached
if (typeof SECRET_SLACK_WEBHOOK_URL !== 'undefined' && SECRET_SLACK_WEBHOOK_URL !== 'default-gh-action-secret') {
await notifySlack(monitor, newMetadata)
}
// Set monitorsOperational and push current day to failedDays
if (!monitorOperational) {
monitorsStateMetadata.monitorsOperational = false
await setKV('s_' + monitor.id, null, newMetadata)
}
// write daily status if monitor is not operational
if (!newMetadata.operational) {
// try to get failed daily status first as KV read is cheaper than write
const kvFailedDayStatusKey = 'h_' + monitor.id + '_' + getDate()
const kvFailedDayStatus = await getKV(kvFailedDayStatusKey)
// write if not found
if (!kvFailedDayStatus) {
console.log('Saving new failed daily status..')
await setKV(kvFailedDayStatusKey, null)
const failedDay = getDate()
if (!monitorsState[monitor.id].failedDays.includes(failedDay)) {
console.log('Saving new failed daily status ...')
monitorsState[monitor.id].failedDays.push(failedDay)
}
}
}
// save last check timestamp including PoP location
// Get Worker PoP and save it to monitorsStateMetadata
const res = await fetch('https://cloudflare-dns.com/dns-query', {
method: 'OPTIONS',
})
const loc = res.headers.get('cf-ray').split('-')[1]
await setKV('lastUpdate', Date.now(), { loc })
monitorsStateMetadata.lastUpdate = {
loc,
time: Date.now()
}
// Save monitorsState and monitorsStateMetadata to KV storage
await setKV('monitors_data', JSON.stringify(monitorsState), monitorsStateMetadata)
return new Response('OK')
}

57
src/functions/helpers.js
View File

@ -2,72 +2,29 @@ import config from '../../config.yaml'
import {useEffect, useState} from 'react'
export async function getMonitors() {
const monitors = await listKV('s_')
return monitors.keys
}
export async function getMonitorsHistory() {
const monitorsHistory = await listKV('h_', 300)
return monitorsHistory.keys
}
export async function getLastUpdate() {
return await getKVWithMetadata('lastUpdate')
}
export async function listKV(prefix = '', cacheTtl = false) {
const cacheKey = 'list_' + prefix + '_' + process.env.BUILD_ID
if (cacheTtl) {
const cachedResponse = await getKV(cacheKey)
if (cachedResponse) {
return JSON.parse(cachedResponse)
}
}
let list = []
let cursor = null
let res = {}
do {
res = await KV_STATUS_PAGE.list({ prefix: prefix, cursor })
list = list.concat(res.keys)
cursor = res.cursor
} while (!res.list_complete)
if (cacheTtl) {
await setKV(cacheKey, JSON.stringify({ keys: list }), null, cacheTtl)
}
return { keys: list }
return await getKVWithMetadata('monitors_data', "json")
}
export async function setKV(key, value, metadata, expirationTtl) {
return KV_STATUS_PAGE.put(key, value, { metadata, expirationTtl })
}
export async function getKV(key, type = 'text') {
return KV_STATUS_PAGE.get(key, type)
export async function getKVWithMetadata(key, type = 'text') {
return KV_STATUS_PAGE.getWithMetadata(key, type)
}
export async function getKVWithMetadata(key) {
return KV_STATUS_PAGE.getWithMetadata(key)
}
export async function deleteKV(key) {
return KV_STATUS_PAGE.delete(key)
}
export async function notifySlack(monitor, newMetadata) {
export async function notifySlack(monitor, operational) {
const payload = {
attachments: [
{
color: newMetadata.operational ? '#36a64f' : '#f2c744',
color: operational ? '#36a64f' : '#f2c744',
blocks: [
{
type: 'section',
text: {
type: 'mrkdwn',
text: `Monitor *${monitor.name}* changed status to *${
newMetadata.operational
operational
? config.settings.monitorLabelOperational
: config.settings.monitorLabelNotOperational
}*`,
@ -79,7 +36,7 @@ export async function notifySlack(monitor, newMetadata) {
{
type: 'mrkdwn',
text: `${
newMetadata.operational ? ':white_check_mark:' : ':x:'
operational ? ':white_check_mark:' : ':x:'
} \`${monitor.method ? monitor.method : "GET"} ${monitor.url}\` - :eyes: <${
config.settings.url
}|Status Page>`,