KEEPER

[00:56:10Z] worker started, pid 897887
[00:56:23Z] heartbeat is 9s old with worker 897887 still running. THIS IS THE SILENT FAILURE.
[00:56:25Z] worker started, pid 897926

# 1. your loop writes one timestamp per tick
node -e 'require("fs").writeFileSync(process.env.HEARTBEAT||"HEARTBEAT", new Date().toISOString())'

# 2. supervise it
sh keeper.sh "node my-loop.mjs"

# 3. ask anytime, from anywhere
node check.mjs
ALIVE  last tick 43s ago  (2026-08-27T00:41:02.118Z)

#!/bin/sh
# grokbot-keeper - keep a Grok Bot's loop alive, and prove it is alive.
#
# THE PROBLEM THIS SOLVES, in the words the community keeps using: "scheduled routines that don't
# fire", "my Bot promised to check back and went quiet". There are two failure modes behind that
# sentence and they need different answers:
#
#   1. THE PROCESS DIED. Easy to catch, easy to fix, and this script restarts it.
#   2. THE PROCESS IS ALIVE AND THE LOOP IS NOT. Much worse, because every check you would
#      naturally run says everything is fine: pgrep finds the pid, the log has no error, the
#      dashboard is green. Only the CLOCK tells the truth. So the worker writes a timestamp every
#      tick and the keeper watches the timestamp, never the pid.
#
# Measured on a real bot: mode 1 cost 42 hours of silence, mode 2 cost 40 minutes during which
# every liveness check passed and nothing had run since the hour before.
#
# USAGE
#   sh keeper.sh "node my-loop.mjs"          start, supervise, restart on exit
#   HEARTBEAT=/tmp/hb sh keeper.sh "..."     custom heartbeat path (default ./HEARTBEAT)
#   STALE_S=600 sh keeper.sh "..."           how old the heartbeat may get before a restart
#
# Your loop must touch the heartbeat file once per tick. One line is enough:
#   node -e 'require("fs").writeFileSync(process.env.HEARTBEAT||"HEARTBEAT", new Date().toISOString())'
#
# No dependencies beyond a POSIX shell. Nothing is sent anywhere. Nothing is installed.

set -eu

CMD=${1:-}
[ -n "$CMD" ] || { echo "usage: sh keeper.sh \"<command that loops>\""; exit 2; }

HEARTBEAT=${HEARTBEAT:-./HEARTBEAT}
STALE_S=${STALE_S:-600}
CHECK_S=${CHECK_S:-60}
LOG=${LOG:-./keeper.log}
export HEARTBEAT

say() { printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$1" | tee -a "$LOG"; }

age_of() {
  # seconds since the heartbeat file was last modified; a missing file is infinitely old
  [ -f "$HEARTBEAT" ] || { echo 999999; return; }
  now=$(date -u +%s)
  # BSD and GNU stat disagree; try both rather than assume the platform
  mtime=$(stat -c %Y "$HEARTBEAT" 2>/dev/null || stat -f %m "$HEARTBEAT" 2>/dev/null || echo 0)
  echo $((now - mtime))
}

start_worker() {
  # setsid where available so the worker survives this shell losing its terminal
  if command -v setsid >/dev/null 2>&1; then
    setsid sh -c "$CMD" >>"$LOG" 2>&1 &
  else
    sh -c "$CMD" >>"$LOG" 2>&1 &
  fi
  WORKER=$!
  say "worker started, pid $WORKER"
}

say "keeper up. command: $CMD"
say "heartbeat: $HEARTBEAT, stale after ${STALE_S}s, checked every ${CHECK_S}s"
start_worker

while :; do
  sleep "$CHECK_S"

  # mode 1: the process is gone
  if ! kill -0 "$WORKER" 2>/dev/null; then
    say "worker $WORKER is gone. restarting."
    start_worker
    continue
  fi

  # mode 2: the process is there and the clock stopped
  A=$(age_of)
  if [ "$A" -gt "$STALE_S" ]; then
    say "heartbeat is ${A}s old with worker $WORKER still running. THIS IS THE SILENT FAILURE."
    kill "$WORKER" 2>/dev/null || true
    sleep 2
    kill -9 "$WORKER" 2>/dev/null || true
    start_worker
  fi
done

#!/usr/bin/env node
// grokbot-keeper/check - answer one question honestly: is my bot actually working?
//
//   node check.mjs                 reads ./HEARTBEAT
//   node check.mjs path/to/file    reads that file
//   node check.mjs --stale 300     what counts as stale, in seconds (default 600)
//
// Exit code 0 alive, 1 stale, 2 never started. Made for cron, CI and a bot checking itself.
//
// WHY A SEPARATE CHECK AT ALL, when the keeper already restarts things: because the owner needs
// an answer without reading a log, and because "alive" and "working" are different claims. This
// prints the age, so a human can see 4 seconds and relax, or see 51 minutes and know that
// whatever the process list says, nothing has run since breakfast.

import { statSync, readFileSync } from "node:fs";

const args = process.argv.slice(2);
const staleIdx = args.indexOf("--stale");
const STALE_S = staleIdx >= 0 ? Number(args[staleIdx + 1]) : 600;
const path = args.find((a) => !a.startsWith("--") && a !== String(STALE_S)) || process.env.HEARTBEAT || "HEARTBEAT";

const human = (s) => (s < 90 ? `${s}s` : s < 5400 ? `${(s / 60).toFixed(1)} min` : `${(s / 3600).toFixed(1)} h`);

let ageS, stamp = null;
try {
  ageS = Math.round((Date.now() - statSync(path).mtimeMs) / 1000);
  // A timestamp INSIDE the file beats the file's mtime when both exist: some filesystems and
  // sync tools touch mtime without the worker having run, and that would make a dead loop look
  // alive. The content is written by the loop itself and cannot lie in that direction.
  try {
    const t = Date.parse(readFileSync(path, "utf8").trim().slice(0, 40));
    if (!Number.isNaN(t)) ageS = Math.max(ageS, Math.round((Date.now() - t) / 1000));
    stamp = readFileSync(path, "utf8").trim().slice(0, 40);
  } catch { /* file unreadable as text, mtime stands */ }
} catch {
  console.log(`NEVER STARTED  no heartbeat at ${path}`);
  console.log(`  The loop has not written once. Check the command the keeper was given.`);
  process.exit(2);
}

if (ageS <= STALE_S) {
  console.log(`ALIVE  last tick ${human(ageS)} ago${stamp ? `  (${stamp})` : ""}`);
  process.exit(0);
}

console.log(`STALE  last tick ${human(ageS)} ago, limit is ${human(STALE_S)}${stamp ? `  (${stamp})` : ""}`);
console.log(`  The process may still be running. That is the point: a live process with a stopped`);
console.log(`  clock is the failure nobody notices. Restart it, or let the keeper do it.`);
process.exit(1);