commit ea1103f52b170055544bafa6d808966336ccdd90 Author: Europium 63 Date: Mon Oct 26 15:44:55 2026 -0700 Initial public release — Inspector v0.7.0 Docker-watchdog with per-container restart rules, Gotify/Pushover notifications, and an embedded dashboard + web settings page. See README.md for an overview and INSTALL.md for the Docker quickstart. diff --git a/.dockerignore b/.dockerignore new file mode 100755 index 0000000..9dbb0a9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.gitignore +.dockerignore +.aider.conf.yml +__pycache__ +*.pyc +.venv +venv +docs +logo +logs/*.log +logs/*.log.* +config.yml +README.md +AGENTS.md +CLAUDE.md +GEMINI.md +CONVENTIONS.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100755 index 0000000..31df90e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,606 @@ +# Changelog + +All notable changes to Inspector are documented here. + +Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [0.7.0] — 2026-10-26 + +### Added + +- **Web settings page** at `/settings` for editing `config.yml` without SSH: + per-section forms for Inspector loop, Gotify, Pushover, Dashboard, + Database, Health summary, Maintenance, Notification mute, and per-container + tuning (thresholds, cooldowns, `restart_on`, color). +- Saves round-trip through ruamel.yaml so comments and key order in + `config.yml` are preserved. Each save keeps a `.bak` of the prior file. + (In the Docker container `.bak` lives in the overlay fs because + `config.yml` is a single-file bind mount; the live file is still updated + cleanly.) +- In-process hot reload via a new `ConfigRef`: edits to container-level + settings and `inspector.dry_run`/`heartbeat`/`interval_seconds` take + effect on the next loop iteration without a container restart. Fields + requiring restart are explicitly labeled in the UI. +- CSRF token on all cookie-authenticated POSTs (Settings forms, + maintenance toggle, mute toggle). Basic Auth clients bypass the check + because they prove identity on every request. +- Secret fields (Gotify token, Pushover token/user_key, dashboard password) + render masked and treat the `__UNCHANGED__` sentinel as "keep existing"; + real secret values are never echoed into HTML. + +### Changed + +- `main.run_loop()` now takes a `ConfigRef`, re-reads config each + iteration, and rebuilds `enabled_configs`/`check_order` so edits apply + without a restart. +- `DashboardServer` constructor simplified: now takes `ConfigRef`, + `states`, `maintenance`, `notification_mute` (per-section configs are + resolved live via `ConfigRef.get()`). +- Added `ruamel.yaml==0.18.6` to `requirements.txt`. + +--- + +## [0.6.0] — 2026-10-26 + +### Added + +- **Pushover notifications** as a second independent channel alongside Gotify. + Enable either, both, or neither under `notifications.pushover`; alerts fan + out to every enabled channel. Supports optional `device` and `sound` + overrides. +- Priority mapping from Inspector's Gotify-style integer scale to Pushover's + `-2..2` range: `<=2 → -1`, `3..4 → 0`, `5..6 → 1`, `>=7 → 2`. Emergency + tier (`2`) ships default `retry`/`expire` values and triggers the usual + Pushover acknowledgement prompt. +- `MultiNotifier` internally fans each `notify()` call out to every enabled + channel; a crash in one channel never prevents the others from running. + +--- + +## [0.5.5] — 2026-06-18 + +### Changed + +- Timestamp display format shortened: year and timezone abbreviation (PST/PDT) removed. + Timestamps now render as `MM-DD HH:MM AM/PM` (e.g. `06-18 09:30 AM`). + +--- + +## [0.5.4] — 2026-06-17 + +### Added + +- Dashboard cards now surface **scheduled maintenance** for containers opted into the recurring + daily window (`maintenance_window: true`): + - while idle, a meta-row `🔧 Next maintenance: in ( – )` shows when + the next window begins, with the window times in the dashboard's local format; + - while the window is active, the phase row shows `🔧 In maintenance — ends in ` + so it's clear when normal monitoring resumes. +- All countdowns reuse the existing 1-second client-side ticker (no page reload needed). +- `/api/status` now returns `maintenance_scheduled` (bool) and `maintenance_window_start` / + `maintenance_window_end` (ISO UTC or null) per container. + +--- + +## [0.5.3] — 2026-06-15 + +### Added + +- Dashboard status cards now show a **phase row** below the error bar, indicating what + Inspector is currently doing for each container: + - `⏱ Cooldown — Xs remaining` — post-restart cooldown is active (countdown to expiry) + - `⏱ Grace period — Xs remaining` — container is booting post-restart; failures not counted + - `⚠ Restart limit — resets in Xs` — hourly restart cap hit; countdown to when it clears + - `⚠ Failing X/Y — next check in Zs` — error count is accumulating toward the restart threshold + - `👁 Observe only` — `restart_on: []` is set; Inspector tracks status but never acts + - `🔧 In maintenance` — container is in a maintenance window + - `Next check in Zs` — container is healthy; soft reminder of the monitoring cadence +- All countdown timers update **every second in the browser** without a page reload (JS + `setInterval`); the existing 30 s auto-refresh keeps data fresh. +- `/api/status` now returns `phase`, `phase_until` (ISO UTC or null), `last_check_at` + (ISO UTC or null), and `interval_seconds` per container entry. + +--- + +## [0.5.2] — 2026-06-15 + +### Added + +- Portainer environments now support `api_token: "ptr_..."` to embed the API token + directly in `config.yml`, as an alternative to `api_token_env` (env var lookup). + `api_token` takes precedence when both are set. Useful when injecting env vars into + the Inspector container is inconvenient. +- Improved token-not-found error message: now mentions both `api_token` and + `api_token_env` options so the fix is self-evident in the logs. +- `_find_stack()` now logs a debug line with how many stacks were scanned and which + `endpoint_id` was used, aiding multi-environment Portainer diagnosis. + +--- + +## [0.5.1] — 2026-06-15 + +### Fixed + +- `health_summary` digest crash: `KeyError: 'portainer_redeploy_failed'` on the + first periodic digest when event history contained Portainer redeploy events. + `health_summary._LABELS` was missing the three Portainer event types + (`portainer_redeploy`, `portainer_redeploy_failed`, `portainer_redeploy_skipped`) + that were added to `SUMMARY_EVENT_TYPES` in v0.4.0. + +--- + +## [0.5.0] — 2026-06-16 + +### Added +- New per-container `recovery_action: restart_then_redeploy` — tiered recovery + for VPN-routed containers. Inspector first tries a plain `docker restart` + (which fixes the common case where the VPN sidecar was merely *restarted*, + keeping the same container id) and only escalates to a Portainer stack + redeploy when the restart errors (e.g. the stale-namespace + "No such container" case after the sidecar is *recreated*) or the container + is still failing on the next recovery cycle. Escalation is non-blocking — it + happens on the following cycle, not via an inline wait. Requires + `portainer_stack_name` like `portainer_stack_redeploy`, and honors the same + `depends_on` gating. + +## [0.4.2] — 2026-06-15 + +### Added +- Result notifications for recovery actions: a successful `docker restart` / + `start` now sends a `✅ … completed` notification, and a successful Portainer + stack redeploy sends `✅ … redeployed`. A failed post-redeploy HTTP recheck + sends `⚠️ … recheck failed`. +- "Testing the Portainer API" guide in the README (uses `python3` + `requests`, + since the container image has no `curl`). + +### Changed +- Portainer API errors now include the HTTP status code and response body in the + failure message/log (e.g. `HTTP 401: {"message":"Invalid JWT token"}`), making + misconfigured tokens/endpoints far easier to diagnose. Added intermediate + debug logging for stack lookup and a warning when a stack name isn't found. + +--- + +## [0.4.1] — 2026-06-15 + +### Fixed +- Per-container `restart_on: []` (empty list) now correctly disables all + restart triggers for that container. Previously, an empty list was + treated as "not set" and silently fell back to the default + `[UNHEALTHY, NOT RUNNING]`, so a container couldn't be configured as + observe-only (e.g. for `depends_on` status tracking without Inspector + restarting it itself). Omitting `restart_on` entirely still defaults to + `[UNHEALTHY, NOT RUNNING]`. + +--- + +## [0.4.0] — 2026-06-15 + +### Added +- New per-container `recovery_action`: `restart_container` (default, + unchanged behavior), `start_container` (calls `docker start` instead of + `docker restart`), and `portainer_stack_redeploy`. +- `portainer_stack_redeploy` recovers containers whose + `network_mode: container:` namespace goes stale after the VPN sidecar + (e.g. Gluetun) is recreated — the `"joining network namespace of container: + No such container"` failure a plain restart can't fix. It redeploys the + owning Portainer stack via the Portainer API. New `integrations.portainer` + config: named environments (token loaded from `api_token_env`, never + logged or stored), `default_environment`, `stack_redeploy_cooldown_seconds`, + `post_redeploy_wait_seconds`. New per-container `portainer_environment` / + `portainer_stack_name`. +- New per-container `depends_on` / `vpn_routed`: containers are checked (and + recovered) in dependency order each cycle, so a VPN sidecar is recovered + before its dependents. Recovery is deferred — without consuming the + container's cooldown — while its `depends_on` container isn't + healthy/starting; `vpn_routed: true` logs why. +- Dry-run extended to cover the new actions, including the Portainer + redeploy flow (`DRY RUN: would resolve/redeploy/wait Portainer ...`). +- New event types: `portainer_redeploy`, `portainer_redeploy_failed`, + `portainer_redeploy_skipped`. +- See [Portainer stack redeploy recovery](README.md#portainer-stack-redeploy-recovery) + in the README. +- New per-container notification mute: `notification_mute` config + (`enabled`, `dir`, `default_hours`, default 12h auto-expiring), dashboard + "Mute notifications"/"Unmute notifications" button + "Muted" badge, + `POST /notifications//mute|unmute`. Muting silences outbound Gotify + notifications only — monitoring, failure counting, restarts/recovery, + cooldowns, event history, and the health-summary digest continue + unaffected. New event types `notification_mute_enter`/`exit`. See + [Muting notifications](README.md#muting-notifications) in the README. + +All new fields are optional and backward-compatible; existing configs are +unaffected. `recovery_action: recreate_container` is **not** implemented and +is rejected at config load. + +--- + +## [0.3.1] — 2026-06-14 + +### Changed +- `http_check` now classifies outcomes into three explicit tiers: the + configured `expected_status` = up, a connection refused / timeout / DNS + failure = unreachable (reported as the synthetic status code `0`), and any + other code = "responding but unexpected". Connection-level failures were + already treated as failed checks (→ `UNREACHABLE`); this makes the + distinction explicit in the result (`HttpCheckResult.status_code`) and in + the log / notification messages. Matching remains exact — the README now + documents how to find a container's actual status code. + +--- + +## [0.3.0] — 2026-06-14 + +### Added +- Opt-in "unreachable" detection per container: even when Docker reports a + container as healthy/running, Inspector can independently verify it's + actually reachable via an HTTP/HTML health check (`http_check`) and/or a + shared [Uptime Kuma](https://github.com/louislam/uptime-kuma) status page + (`integrations.uptime_kuma` + per-container `uptime_kuma.monitor_name`). + A failing check on an otherwise-healthy container surfaces as a new + `UNREACHABLE` status (red badge on the dashboard); opt it into a + container's `restart_on` to have it count toward failures/restarts like + any other status. Both checks default to disabled and never crash + Inspector. +- Events page (`/events`) redesigned as a daily view: shows one calendar + day (12:00 AM-11:59 PM US Pacific) at a time, defaulting to today, with + "Prev day" / "Next day" navigation (`?date=YYYY-MM-DD`, also supported by + `/api/events`). The timeline now always renders all 24 hourly markers + (`12 AM`..`11 PM`) so you can scroll directly to a time of day, plus a + trailing "11 PM (yesterday)" section for the 1-hour lookback overlap from + the previous day. + +### Changed +- All dashboard and Gotify notification timestamps now render in US + Pacific time (`PDT`/`PST`, DST-aware) instead of UTC, e.g. + `2026-06-14 03:45 PM PDT`. JSON API endpoints are unaffected and continue + to return raw UTC ISO 8601 timestamps. + +--- + +## [0.2.6] — 2026-06-14 + +### Changed +- Gotify notifications reformatted as Markdown (`extras.client::display.contentType: + text/markdown`) with richer context per message: error counts and + thresholds, previous/current status, Docker error details for failed + restarts, restart/cooldown info, and a formatted timestamp. The + health-history digest is now a Markdown bulleted list with a header. +- Dashboard timestamps (container "Last restart" / "Cooldown until", and + the events timeline) now render as 12-hour time with AM/PM (e.g. + `2026-06-14 03:45 PM UTC`) instead of raw ISO 8601. + +### Fixed +- "Last restart" no longer reverts to "Never" after Inspector itself + restarts: on startup, with `database.enabled: true`, each container's + most recent recorded restart (and any still-active cooldown) is restored + from the event history database. + +--- + +## [0.2.5] — 2026-06-14 + +### Added +- Per-container identity colors on the dashboard: each container card gets + a soft background tint and colored name from a 16-color auto-assigned + palette, and the same color carries over to that container's chip on the + events timeline. Override the auto-assigned color with an optional + `color: "#rrggbb"` field per container in `config.yml`. + +--- + +## [0.2.4] — 2026-06-14 + +### Fixed +- Sign out: removed the redirect-if-authenticated on `GET /login` so sign + out always reaches the login form, even with a cached browser Basic Auth + session. +- Mobile header: version/refresh text now hides on narrow screens, and + sign-out renders as a small outlined button beside the brand. +- Status page error bar: shows 100% green when a container has zero errors + (previously rendered as an invisible 0-width bar) and a proportional red + fill once errors are detected. +- Container cards: hide the cooldown row when no cooldown is active, and + show "Never" instead of "—" when a container has no restart history. +- Events page: the "event history not enabled" message now includes the + `database: enabled: true` config snippet needed to turn it on. + +--- + +## [0.2.3] — 2026-06-14 + +### Changed +- Replaced the browser-native Basic Auth prompt with a styled dark-theme + login page backed by a session cookie (24h expiry); HTTP Basic Auth still + works for API clients. Added a "Sign out" link, and made the dashboard + header, stats grid, and card grid responsive on mobile. + +--- + +## [0.2.2] — 2026-06-14 + +### Added +- Dashboard pages now set a full favicon/app-icon set (favicon.ico, + favicon.svg, favicon-96x96.png, apple-touch-icon.png, web app manifest + icons, site.webmanifest) generated from the Inspector icon. Served from + new `GET /static/` endpoints alongside the existing icon route + (Basic Auth required, same as every other dashboard route). + +--- + +## [0.2.1] — 2026-06-14 + +### Changed +- Web dashboard redesigned: dark "card grid" theme. Each container gets its + own card (status badge, error-count progress bar, cooldown/last-restart + details, maintenance badge and toggle button) instead of a plain table; + the events page renders a colour-coded vertical timeline instead of a + table. The header now shows the Inspector icon plus an "Inspector" / + "Monitor. Restart. Report." wordmark set in + [Google Sans Flex](https://fonts.google.com/specimen/Google+Sans+Flex) + (loaded from Google Fonts; falls back to the system sans-serif if + unavailable). New `GET /static/icon.png` endpoint (Basic Auth required, + same as every other dashboard route) serves the icon from a new + `src/inspector/assets/` package directory. + +--- + +## [0.2.0] — 2026-06-14 + +### Added +- Optional embedded web dashboard (`dashboard.enabled`, default `false`), + protected by HTTP Basic Auth (`dashboard.username` / `dashboard.password`). + If enabled with an empty password, Inspector logs an error and refuses to + start the dashboard (fails closed). Runs in the same process as the + monitoring loop — no extra container, zero new dependencies (stdlib + `http.server`). + - **Status page (`/`, JSON at `/api/status`)** — live per-container table: + current status, error count vs. threshold, cooldown expiry, last restart + time, and whether it's in a maintenance window. Auto-refreshes every + 30 seconds. + - **Events page (`/events`, JSON at `/api/events`)** — last 100 rows from + event history, if `database.enabled: true`. + - **Maintenance toggle** — when `maintenance.enabled: true`, per-container + "Enter/Exit maintenance" buttons `POST /maintenance//enter|exit`, + touching/removing the same marker files described under "Ad-hoc marker + files". Returns `409` if maintenance windows are disabled, `404` for an + unknown container. + New module `src/inspector/dashboard.py`, new config section `dashboard:` + in `config.example.yml`. +- `MaintenanceTracker.peek(name)` — read-only maintenance check (no + transition logging, no eventstore writes) used by the dashboard. +- `tests/test_dashboard.py` — unit tests covering the disabled no-op, the + empty-password refusal, Basic Auth enforcement, the status page and API, + the maintenance toggle (enter/exit, disabled-config 409, unknown-container + 404), and the events page/API with and without event history enabled. + +--- + +## [0.1.11] — 2026-06-14 + +### Added +- Optional maintenance windows (`maintenance.enabled`, default `false`). + Two complementary mechanisms, both gated by this one flag: + - **Recurring daily window** — containers with `maintenance_window: true` + are "in maintenance" every day between `maintenance.start` and + `maintenance.end` (default `00:00`–`06:00`, local time via `TZ`, may + wrap past midnight). + - **Ad-hoc marker files** — `touch`/`rm` + `/` (default + `/app/data/maintenance/`) for unplanned maintenance. A + marker older than `maintenance.max_hours` (default `8`) auto-expires. + + While either mechanism is active for a container, Inspector is fully + silent for it: no failure counting, restarts, notifications, + status-change logging, or event recording. Only one-time + "entering/exiting maintenance window" log lines and new + `maintenance_enter`/`maintenance_exit` events (excluded from the + health-history digest) mark the transitions. State (error counts, + cooldowns, restart history) carries over unchanged across maintenance. + New module `src/inspector/maintenance.py`, new config section + `maintenance:` in `config.example.yml`, new per-container + `maintenance_window` field. +- `tests/test_maintenance.py` — unit tests covering the disabled no-op, + marker-file activation and auto-expiry, the recurring window (including + midnight wraparound), enter/exit transition logging and event recording, + and safe handling of marker-check errors. + +--- + +## [0.1.10] — 2026-06-14 + +### Added +- Optional periodic health-history digest (`health_summary.enabled`, + default `false`). When enabled, Inspector logs a per-container summary + of notable activity (failures, restarts, restart-limit hits, recoveries, + not-found events — status changes are excluded) since the last digest, + every `health_summary.interval_hours` (default `24`). The first digest + fires immediately on startup, covering the last `interval_hours`. The + digest is always logged; it is pushed to Gotify (at + `health_summary.priority`, default `3`) only if any container had + activity in the window. Depends on `database.enabled: true` for real + data — if `health_summary.enabled` is `true` but event history isn't + enabled, Inspector logs a one-time startup warning and digests report + "no issues" for every container. New module + `src/inspector/health_summary.py`, new config section + `health_summary:` in `config.example.yml`. +- `EventStore.summarize(since)` — per-container counts of notable event + types at/after a timestamp, and `EventStore.enabled` property. +- `tests/test_health_summary.py` — unit tests covering the disabled + no-op, the database-disabled warning, immediate first report with a + capped window, interval gating, message formatting, and Gotify + activity gating. + +--- + +## [0.1.9] — 2026-06-13 + +### Added +- Optional SQLite event history (`database.enabled`, default `false`). + When enabled, Inspector persists notable per-container events (status + changes, recoveries, failures, restarts, restart-limit hits, + container-not-found) to a local SQLite database at `database.path` + (default `/app/data/events.db`), surviving container restarts. Old + events are pruned according to `database.retention_days` (default 30) + on startup and once per 24h thereafter. The database is opened in WAL + mode for future concurrent read access (e.g. a read-only dashboard). + A database error is logged but never crashes Inspector. New module + `src/inspector/eventstore.py`, new config section `database:` in + `config.example.yml`, new `./data` volume mount in `compose.yml` / + `compose.prod.yml`, new `/app/data` directory in `Dockerfile`. +- `tests/test_eventstore.py` — unit tests covering the disabled no-op, + row insertion, WAL mode, and retention pruning. + +### Changed +- `_check_one()` signature gains an `eventstore: EventStore` parameter + (always passed; no-ops internally when `database.enabled` is `false`), + mirroring the existing `notifier: GotifyNotifier` pattern. +- README "What Inspector does not do" updated: live `ContainerState` is + still in-memory and resets on restart, but event *history* can now + persist via the new opt-in SQLite database. + +--- + +## [0.1.8] — 2026-06-13 + +### Changed +- Log format overhauled for readability: container names are now + left-padded into a fixed-width column, the redundant logger-name field + (`inspector:`) was dropped from both the file and console formatters, and + a blank line now brackets each failure → restart → recovery "incident" so + it stands out as a block. Message wording adjusted accordingly (e.g. + `gotify Restarting due to 3 consecutive UNHEALTHY checks` instead of + `Restarting gotify due to 3 consecutive UNHEALTHY checks`). + +--- + +## [0.1.7] — 2026-06-13 + +### Fixed +- Docker socket permission errors at startup no longer crash with a raw + Python traceback. `run_loop()` now catches the connection failure and + logs an actionable message pointing at `DOCKER_GID` + (`stat -c '%g' /var/run/docker.sock`). + +--- + +## [0.1.6] — 2026-06-13 + +### Added +- `tests/test_config.py` — 4 unit tests covering config loading, defaults, + and validation (non-mapping containers rejected, unknown `restart_on` + statuses rejected). +- `tests/test_health.py` — 8 unit tests covering all Docker status + classifications (HEALTHY, UNHEALTHY, STARTING, NO HEALTH CHECK, + NOT RUNNING, UNKNOWN, missing state, ERROR constant). +- `tests/test_state.py` — 10 unit tests covering failure counting, restart + decision logic, and the full per-container state machine (dry-run, + grace period, cooldown, hourly restart limit, recovery). + +### Changed +- CI workflow now runs the full unit test suite + (`python3 -m unittest discover -s tests -v`) before building the image. + +--- + +## [0.1.5] — 2026-06-13 + +### Changed +- CI and release workflows reverted to `runs-on: ubuntu-latest` (the + self-hosted runner approach introduced in v0.1.3 was abandoned — the + Gitea-hosted runner handles builds reliably). +- Gitea runner setup guide (`docs/gitea_runner_setup.md`, + `ops/gitea-runner/`) updated with label-mismatch troubleshooting and + clearer registration instructions. + +### Added +- README documents the verified release baseline (v0.1.5) and the + step-by-step release flow. + +--- + +## [0.1.4] — 2026-06-13 + +### Added +- Inspector logo and wordmark assets: `logo/inspector_icon.{png,svg}`, + `logo/inspector_wordmark.{png,svg}`, `logo/inspector_icon_chip.{png,svg}`. +- README header now displays the Inspector wordmark image. + +### Changed +- Previous logo files moved to `logo/archive/`. +- `.gitignore` — added `act_data/` (local `act` runner data directory). + +--- + +## [0.1.3] — 2026-06-13 + +### Changed +- CI and release workflows switched to `runs-on: self-hosted`. +- CI simplified to use `docker build` directly (removed + `docker/setup-buildx-action` dependency). + +--- + +## [0.1.2] — 2026-06-13 + +### Changed +- Release workflow rewritten to use native `docker login` / `docker push` + shell commands, replacing the `docker/build-push-action` third-party + action. + +### Added +- `workflow_dispatch` trigger on the release workflow — enables manual + image builds without a tag push (useful for testing the CI publish path). + +--- + +## [0.1.1] — 2026-06-13 + +### Fixed +- Release workflow secret names corrected: + `GITEA_REGISTRY_USER` → `REGISTRY_USER`, + `GITEA_REGISTRY_TOKEN` → `REGISTRY_TOKEN`. + The v0.1.0 release workflow referenced the wrong secret names, causing + the publish step to fail. + +--- + +## [0.1.0] — 2026-06-13 + +### Added +- Initial Inspector v1: self-hosted Python Docker container health watcher. +- Multi-container support via a single `config.yml` — replaces the old + pattern of one script + one systemd service per watched container. +- Per-container state machine with independent failure counters. +- Health status classification: HEALTHY, UNHEALTHY, STARTING, + NO HEALTH CHECK, NOT RUNNING, CONTAINER NOT FOUND, ERROR, UNKNOWN. +- Restart logic: fires after `max_error_count` consecutive failures. +- Cooldown: post-restart window that blocks further restarts. +- Grace period: post-restart window that suppresses failure counting while + a container boots. +- Hourly restart cap (`max_restarts_per_hour`) to break endless loops; + fires a critical Gotify alert when hit. +- Optional Gotify notifications on state transitions (not per-loop). +- Dry-run mode: logs decisions without executing restarts. +- Rotating log file (`TimedRotatingFileHandler`, midnight, 30-day retention) + with colored console output and plain-text file output. +- `Dockerfile` — `python:3.12-slim` base, non-root `inspector` user + (UID/GID 1000). +- `compose.yml` — dev compose: builds from source, mounts Docker socket. +- `compose.prod.yml` — prod compose: pulls published image from Gitea + registry, no local build. +- `requirements.txt` — pinned: `docker==7.1.0`, `PyYAML==6.0.2`, + `requests==2.32.3`, `colorlog==6.9.0`. +- `Makefile` with `build`, `smoke`, `compose-config`, `tag`, `push-local`, + `release`, `clean`, `version`, `help` targets. +- Gitea Actions CI (`ci.yml`): byte-compile + Docker build on every push. +- Gitea Actions release (`release.yml`): build and publish four image tags + on `v*.*.*` tag push. +- `scripts/bump_version.sh` — updates `__version__` in `src/inspector/__init__.py`. +- `scripts/smoke_test.sh` — end-to-end dry-run test against the live + Docker socket. +- README with full feature documentation, configuration reference, + Portainer deployment guide, and troubleshooting table. diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..b884186 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY src/ /app/src/ +RUN chmod -R a+rX /app/src + +ENV PYTHONPATH=/app/src + +ARG INSPECTOR_UID=1000 +ARG INSPECTOR_GID=1000 +RUN groupadd --gid ${INSPECTOR_GID} inspector \ + && useradd --uid ${INSPECTOR_UID} --gid inspector --home-dir /app --shell /usr/sbin/nologin inspector \ + && mkdir -p /app/logs /app/data \ + && chown -R inspector:inspector /app/logs /app/data + +USER inspector + +CMD ["python", "-m", "inspector.main", "--config", "/app/config.yml"] diff --git a/INSTALL.md b/INSTALL.md new file mode 100755 index 0000000..ef6ccec --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,70 @@ +# Install + +Inspector watches Docker containers, restarts the ones you tell it to, +and notifies you via Gotify and/or Pushover. All configuration lives in +a single `config.yml` that you can edit by hand or from the built-in web +settings page. + +## Requirements + +- Docker Engine 24+ and `docker compose` v2. +- A machine that can reach `/var/run/docker.sock` (any Docker host). +- Optional: a Gotify server and/or a Pushover account for notifications. + +## Quickstart + +```bash +# 1. Grab the compose file and example config +mkdir -p ~/docker/inspector && cd ~/docker/inspector +curl -L -o compose.yml https://gitea.euronvault.com/euronvault/inspector/raw/branch/main/compose.yml +curl -L -o config.yml https://gitea.euronvault.com/euronvault/inspector/raw/branch/main/config.example.yml + +# 2. Edit config.yml — at minimum, list the containers you want watched +# under the `containers:` key. +$EDITOR config.yml + +# 3. Set your Docker group GID so the container can read the socket. +echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" > .env + +# 4. Pull and run +docker compose pull +docker compose up -d +docker compose logs -f inspector +``` + +If `dashboard.enabled: true` in your config, visit +`http://:8080/` after startup to see the dashboard, and +`http://:8080/settings` to edit the config from the browser. + +## Pinning a version + +Prefer an immutable tag over `latest` in production: + +```bash +# .env +INSPECTOR_TAG=0.7.0 +``` + +Images are published at `gitea.euronvault.com/euronvault/inspector`. + +## Upgrading + +```bash +docker compose pull +docker compose up -d +``` + +Your `config.yml` is persisted outside the container, so upgrades don't +touch it. The settings page keeps a `.bak` of the previous file next to +`config.yml` on every save (when the directory is bind-mounted; see the +README for the Docker caveat). + +## Uninstall + +```bash +docker compose down +rm -rf logs data config.yml compose.yml .env +``` + +Inspector doesn't store anything outside the directory it was started +from. diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000..89c7ad8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Europium 63 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100755 index 0000000..e1a1b0e --- /dev/null +++ b/README.md @@ -0,0 +1,139 @@ +# Inspector + +Inspector is a small, opinionated Docker-watchdog. It polls the Docker +daemon, applies per-container restart rules, pings Gotify and/or +Pushover when things go wrong, and gives you a dashboard that shows +what's happening right now and what happened earlier today. + +Built for home-lab and small-fleet operators who want "it just works" +container recovery without standing up a full observability stack. + +## What Inspector does + +- Polls each configured container on a fixed interval and classifies it: + `HEALTHY`, `UNHEALTHY`, `STARTING`, `NOT RUNNING`, `ERROR`, + `UNREACHABLE`, `CONTAINER NOT FOUND`, `NO HEALTH CHECK`, `UNKNOWN`. +- Restarts a container when it crosses a configured error threshold — + with cooldowns, grace periods, and an hourly restart cap so a flapping + container doesn't DDoS your host. +- Opt-in HTTP reachability checks per container (useful for VPN-routed + containers whose tunnel drops while Docker still reports "healthy"). +- Opt-in Uptime Kuma integration as a second reachability source. +- Advanced recovery via Portainer stack redeploy for containers whose + network namespace is tied to a VPN sidecar. +- Sends notifications on transitions, not on every poll — one line when + something breaks, one line when it recovers. +- Optional SQLite event history + a daily/weekly digest. +- Scheduled maintenance windows (don't act during 2–6 AM). +- Manual notification mute per container, from the dashboard. +- Embedded web dashboard + events timeline + **web settings page** for + editing `config.yml` without SSH. + +## What Inspector is not + +- Not a metrics system. It doesn't scrape Prometheus or push + dashboards; use Grafana / Netdata / your real monitoring stack for + that. +- Not an orchestrator. It only acts on containers you've explicitly + listed in `config.yml`. +- Not multi-host. One Inspector per Docker socket. + +## Install + +See [`INSTALL.md`](./INSTALL.md). TL;DR: + +```bash +mkdir -p ~/docker/inspector && cd ~/docker/inspector +curl -L -o compose.yml https://gitea.euronvault.com/euronvault/inspector/raw/branch/main/compose.yml +curl -L -o config.yml https://gitea.euronvault.com/euronvault/inspector/raw/branch/main/config.example.yml +$EDITOR config.yml +echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" > .env +docker compose up -d +``` + +## Configuration + +Everything lives in `config.yml` next to the compose file. Start from +[`config.example.yml`](./config.example.yml) — every key is commented. +Highlights: + +- `inspector.interval_seconds` — poll cadence. +- `inspector.dry_run` — classify + log, never restart. +- `notifications.gotify` / `notifications.pushover` — enable either, + both, or neither. +- `containers..max_error_count` / `cooldown_seconds` / + `grace_period_seconds` / `max_restarts_per_hour` — the restart knobs. +- `containers..restart_on` — which statuses trigger a restart + (leave empty for observe-only). +- `containers..http_check` — opt-in HTTP ping. +- `containers..recovery_action` — `restart_container` (default) + or the advanced Portainer-based options. + +### Dashboard and settings page + +If `dashboard.enabled: true`, Inspector exposes a small web UI on +`:8080`: + +- `/` — container cards with current status, error bar, phase + (cooldown / grace / failing / observe only / maintenance), and + maintenance / mute toggles. +- `/events` — per-day timeline of status changes, restarts, recoveries, + and failures (requires `database.enabled: true`). +- `/settings` — forms for every top-level config section plus one card + per container. Saves preserve comments and key order in `config.yml` + and keep a `.bak`. Fields that take effect on the next loop iteration + (per-container thresholds, `dry_run`, `heartbeat`, `interval_seconds`) + hot-apply; fields bound at startup are labeled **requires restart**. + +Note on `.bak` inside Docker: when `config.yml` is bind-mounted as a +single file (the compose default), the `.bak` lives in the container's +overlay fs — the live file still updates cleanly, but the backup is +lost when the container is removed. To keep the `.bak` on the host, +bind-mount the directory instead of the single file. + +### Notifications + +Both channels are independent. If both are enabled, every alert goes to +both. Inspector sends on transitions: + +``` +🚨 first failure threshold crossed +🔁 container restart triggered +✅ restart / start completed +✅ container recovered +🛑 restart failed / hourly limit exceeded +❓ configured container not found +``` + +Gotify messages are Markdown; Pushover doesn't render Markdown so the +`**bold**` passthrough is cosmetic — bodies are identical. + +Pushover priorities map from Inspector's Gotify-style scale (default 5) +to Pushover's `-2..2` range: `<=2 → -1`, `3..4 → 0`, `5..6 → 1`, +`>=7 → 2`. Priority 2 (emergency) ships default `retry` / `expire` and +your Pushover client prompts for acknowledgement. + +## Docker socket security + +Inspector talks to `/var/run/docker.sock`. That socket is effectively +root on your host: anything that can write to it can start, stop, +inspect, and (through bind mounts) read any file as root. Inspector +mounts it read-write because it needs to restart containers. + +Follow the usual home-lab precautions: + +- Don't publish the dashboard port on an untrusted network. Use a + reverse proxy with auth, a tailnet, or SSH tunneling. +- Set a strong `dashboard.password`. +- Store Portainer / Gotify / Pushover tokens in the config file on a + host-only path with permissions `600`. + +## Versioning + +Semantic versioning (`MAJOR.MINOR.PATCH`). Image tags are published for +every release; `latest` tracks the newest tag. Pin a specific version +in production — see `INSTALL.md`. + +## License + +[MIT](./LICENSE). diff --git a/compose.yml b/compose.yml new file mode 100755 index 0000000..1f5e391 --- /dev/null +++ b/compose.yml @@ -0,0 +1,29 @@ +# Inspector compose file for buddies / self-hosters. +# Pulls the published image from gitea.euronvault.com. +# +# Before `docker compose up -d`: +# 1. Copy config.example.yml -> config.yml and edit it. +# 2. Set DOCKER_GID to your host's docker group GID: +# getent group docker | cut -d: -f3 +# Export it in your shell or put it in a .env file next to this file. +# 3. Optionally set INSPECTOR_TAG to pin a version +# (default: latest). Prefer a pinned X.Y.Z for production. +services: + inspector: + image: gitea.euronvault.com/euronvault/inspector:${INSPECTOR_TAG:-latest} + container_name: inspector + restart: unless-stopped + # GID of the docker group on the host so the non-root user inside the + # container can read /var/run/docker.sock. Required. + group_add: + - "${DOCKER_GID:-999}" + ports: + # Only published if dashboard.enabled is true in config.yml. + - "${INSPECTOR_PORT:-8080}:8080" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./config.yml:/app/config.yml + - ./logs:/app/logs + - ./data:/app/data + environment: + - TZ=${TZ:-UTC} diff --git a/config.example.yml b/config.example.yml new file mode 100755 index 0000000..b7f77be --- /dev/null +++ b/config.example.yml @@ -0,0 +1,137 @@ +# Inspector example configuration. +# Copy to config.yml, edit, and mount into the container at /app/config.yml. + +inspector: + interval_seconds: 60 + log_level: INFO + dry_run: false + log_dir: /app/logs + log_filename: inspector.log + log_retention_days: 30 + heartbeat: false + +notifications: + # Gotify: https://gotify.net + gotify: + enabled: false + url: "" + token: "" + priority: 5 + + # Pushover: https://pushover.net + # Independent of Gotify — enable either, both, or neither. If both are + # enabled, every alert is sent to both. Inspector's priority is an + # integer on a Gotify-style scale (default 5) and is mapped to + # Pushover's -2..2 range at send time: <=2 -> -1, 3..4 -> 0, + # 5..6 -> 1, >=7 -> 2. + pushover: + enabled: false + token: "" # Pushover application API token + user_key: "" # Pushover user or group key + priority: 5 + device: "" # optional target device; blank = all devices + sound: "" # optional sound name; blank = user default + +# Persist a durable per-container event history to a local SQLite DB. +# Enables the /events page and the Health summary digest. +database: + enabled: false + path: /app/data/events.db + retention_days: 30 + +# Periodic digest of per-container events, delivered through the enabled +# notification channels. Requires database.enabled: true to have data. +health_summary: + enabled: false + interval_hours: 24 + priority: 3 + +# Recurring daily maintenance window. Containers that opt in with +# `maintenance_window: true` are not acted on during this window. +maintenance: + enabled: false + start: "00:00" + end: "06:00" + dir: /app/data/maintenance + max_hours: 8 + +# Manual notification muting per container (toggled from the dashboard). +notification_mute: + enabled: false + dir: /app/data/muted + default_hours: 12 + +# Embedded web dashboard. Enable with a non-empty password. +# Visit http://:/ for status, /events for history, +# /settings to edit this file from the browser. +dashboard: + enabled: false + host: 0.0.0.0 + port: 8080 + username: admin + password: "" + +# Optional external integrations. +integrations: + # Uptime Kuma: optionally use a public status page as an extra + # reachability check per container. + uptime_kuma: + enabled: false + url: "https://status.example.com" + status_page_slug: "default" + + # Portainer API integration: used by the advanced recovery actions + # `portainer_stack_redeploy` and `restart_then_redeploy`, which can + # recover containers whose network namespace is tied to a VPN sidecar. + # Leave disabled unless you need it. + # + # Token options (pick one per environment): + # api_token_env: "ENV_VAR_NAME" — reads token from that env var at + # runtime (preferred; not stored in config) + # api_token: "ptr_..." — embed token directly in this file + portainer: + enabled: false + default_environment: local_docker + stack_redeploy_cooldown_seconds: 300 + post_redeploy_wait_seconds: 90 + environments: + local_docker: + base_url: "http://portainer.example.local:9000" + api_token_env: "PORTAINER_API_TOKEN" + endpoint_id: 1 + verify_ssl: false + +containers: + # Minimal entry: Inspector only acts on containers listed here. + # Each key is the Docker container name on your host. + example-web: + enabled: true + max_error_count: 5 + cooldown_seconds: 300 + grace_period_seconds: 120 + max_restarts_per_hour: 3 + restart_on: + - UNHEALTHY + - NOT RUNNING + - ERROR + + # Example with an opt-in HTTP reachability check. Useful when Docker + # reports a container as healthy but it's not actually serving + # requests (common with VPN-routed containers whose tunnel drops). + example-api: + enabled: true + max_error_count: 5 + cooldown_seconds: 300 + grace_period_seconds: 120 + max_restarts_per_hour: 3 + color: "#60a5fa" + restart_on: + - UNHEALTHY + - NOT RUNNING + - ERROR + - UNREACHABLE + http_check: + enabled: true + url: "http://example-api:8080/health" + timeout_seconds: 5 + expected_status: 200 diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000..253a993 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +docker==7.1.0 +PyYAML==6.0.2 +requests==2.32.3 +colorlog==6.9.0 +ruamel.yaml==0.18.6 diff --git a/src/inspector/__init__.py b/src/inspector/__init__.py new file mode 100755 index 0000000..49e0fc1 --- /dev/null +++ b/src/inspector/__init__.py @@ -0,0 +1 @@ +__version__ = "0.7.0" diff --git a/src/inspector/assets/apple-touch-icon.png b/src/inspector/assets/apple-touch-icon.png new file mode 100755 index 0000000..f931a4e Binary files /dev/null and b/src/inspector/assets/apple-touch-icon.png differ diff --git a/src/inspector/assets/favicon-96x96.png b/src/inspector/assets/favicon-96x96.png new file mode 100755 index 0000000..44986c3 Binary files /dev/null and b/src/inspector/assets/favicon-96x96.png differ diff --git a/src/inspector/assets/favicon.ico b/src/inspector/assets/favicon.ico new file mode 100755 index 0000000..c85b112 Binary files /dev/null and b/src/inspector/assets/favicon.ico differ diff --git a/src/inspector/assets/favicon.svg b/src/inspector/assets/favicon.svg new file mode 100755 index 0000000..9fa4569 --- /dev/null +++ b/src/inspector/assets/favicon.svg @@ -0,0 +1 @@ +RealFaviconGeneratorhttps://realfavicongenerator.net \ No newline at end of file diff --git a/src/inspector/assets/inspector_icon_outline.png b/src/inspector/assets/inspector_icon_outline.png new file mode 100755 index 0000000..e9e5e25 Binary files /dev/null and b/src/inspector/assets/inspector_icon_outline.png differ diff --git a/src/inspector/assets/site.webmanifest b/src/inspector/assets/site.webmanifest new file mode 100755 index 0000000..2076f60 --- /dev/null +++ b/src/inspector/assets/site.webmanifest @@ -0,0 +1,21 @@ +{ + "name": "Inspector", + "short_name": "Inspector", + "icons": [ + { + "src": "/static/web-app-manifest-192x192.png", + "sizes": "192x192", + "type": "image/png", + "purpose": "maskable" + }, + { + "src": "/static/web-app-manifest-512x512.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "maskable" + } + ], + "theme_color": "#0b1220", + "background_color": "#0b1220", + "display": "standalone" +} diff --git a/src/inspector/assets/web-app-manifest-192x192.png b/src/inspector/assets/web-app-manifest-192x192.png new file mode 100755 index 0000000..1273bc9 Binary files /dev/null and b/src/inspector/assets/web-app-manifest-192x192.png differ diff --git a/src/inspector/assets/web-app-manifest-512x512.png b/src/inspector/assets/web-app-manifest-512x512.png new file mode 100755 index 0000000..42822b9 Binary files /dev/null and b/src/inspector/assets/web-app-manifest-512x512.png differ diff --git a/src/inspector/config.py b/src/inspector/config.py new file mode 100755 index 0000000..863537c --- /dev/null +++ b/src/inspector/config.py @@ -0,0 +1,502 @@ +from __future__ import annotations + +import io +import os +import shutil +import threading +from dataclasses import dataclass, field +from pathlib import Path + +import yaml +from ruamel.yaml import YAML + + +VALID_STATUSES = { + "HEALTHY", + "UNHEALTHY", + "STARTING", + "NO HEALTH CHECK", + "NOT RUNNING", + "CONTAINER NOT FOUND", + "ERROR", + "UNKNOWN", + "UNREACHABLE", +} + +VALID_RECOVERY_ACTIONS = { + "restart_container", + "start_container", + "portainer_stack_redeploy", + "restart_then_redeploy", +} + + +@dataclass(frozen=True) +class InspectorConfig: + interval_seconds: int = 60 + log_level: str = "INFO" + dry_run: bool = False + log_dir: str = "/app/logs" + log_filename: str = "inspector.log" + log_retention_days: int = 30 + heartbeat: bool = False + + +@dataclass(frozen=True) +class GotifyConfig: + enabled: bool = False + url: str = "" + token: str = "" + priority: int = 5 + + +@dataclass(frozen=True) +class PushoverConfig: + enabled: bool = False + token: str = "" # Pushover application API token + user_key: str = "" # Pushover user or group key + priority: int = 5 # Inspector-scale priority, mapped to Pushover's -2..2 at send time + device: str = "" # optional target device name; blank = all devices + sound: str = "" # optional sound name; blank = user default + + +@dataclass(frozen=True) +class DatabaseConfig: + enabled: bool = False + path: str = "/app/data/events.db" + retention_days: int = 30 + + +@dataclass(frozen=True) +class HealthSummaryConfig: + enabled: bool = False + interval_hours: int = 24 + priority: int = 3 + + +@dataclass(frozen=True) +class MaintenanceConfig: + enabled: bool = False + start: str = "00:00" + end: str = "06:00" + dir: str = "/app/data/maintenance" + max_hours: int = 8 + + +@dataclass(frozen=True) +class NotificationMuteConfig: + enabled: bool = False + dir: str = "/app/data/muted" + default_hours: int = 12 + + +@dataclass(frozen=True) +class DashboardConfig: + enabled: bool = False + host: str = "0.0.0.0" + port: int = 8080 + username: str = "admin" + password: str = "" + + +@dataclass(frozen=True) +class HttpCheckConfig: + enabled: bool = False + url: str = "" + timeout_seconds: float = 5.0 + expected_status: int = 200 + + +@dataclass(frozen=True) +class UptimeKumaMonitorConfig: + enabled: bool = False + monitor_name: str = "" + + +@dataclass(frozen=True) +class UptimeKumaConfig: + enabled: bool = False + url: str = "" + status_page_slug: str = "" + + +@dataclass(frozen=True) +class PortainerEnvironmentConfig: + base_url: str = "" + api_token_env: str = "" # name of env var holding the token (preferred; token never stored) + api_token: str = "" # token embedded directly in config (use api_token_env for security) + endpoint_id: int = 0 + verify_ssl: bool = True + + +@dataclass(frozen=True) +class PortainerConfig: + enabled: bool = False + default_environment: str = "" + stack_redeploy_cooldown_seconds: int = 300 + post_redeploy_wait_seconds: int = 90 + environments: dict[str, PortainerEnvironmentConfig] = field(default_factory=dict) + + +@dataclass(frozen=True) +class IntegrationsConfig: + uptime_kuma: UptimeKumaConfig + portainer: PortainerConfig + + +@dataclass(frozen=True) +class ContainerConfig: + name: str + enabled: bool = True + max_error_count: int = 5 + cooldown_seconds: int = 300 + grace_period_seconds: int = 120 + max_restarts_per_hour: int = 3 + restart_on: frozenset[str] = field(default_factory=lambda: frozenset({"UNHEALTHY", "NOT RUNNING"})) + maintenance_window: bool = False + color: str = "" # optional dashboard accent color, e.g. "#60a5fa" + http_check: HttpCheckConfig = field(default_factory=HttpCheckConfig) + uptime_kuma: UptimeKumaMonitorConfig = field(default_factory=UptimeKumaMonitorConfig) + depends_on: str = "" + vpn_routed: bool = False + recovery_action: str = "" # "" -> default restart_container behavior + portainer_environment: str = "" + portainer_stack_name: str = "" + + +@dataclass(frozen=True) +class AppConfig: + inspector: InspectorConfig + gotify: GotifyConfig + pushover: PushoverConfig + database: DatabaseConfig + health_summary: HealthSummaryConfig + maintenance: MaintenanceConfig + notification_mute: NotificationMuteConfig + dashboard: DashboardConfig + integrations: IntegrationsConfig + containers: tuple[ContainerConfig, ...] + + +def load_config(path: str | Path) -> AppConfig: + raw = yaml.safe_load(Path(path).read_text()) or {} + return build_config_from_raw(raw) + + +def build_config_from_raw(raw: dict) -> AppConfig: + """Validate and materialize an AppConfig from an already-parsed dict. + + Used both by load_config() on startup and by the settings-page save path + to validate an edited raw dict before writing it back to disk. + """ + insp_raw = raw.get("inspector", {}) or {} + inspector = InspectorConfig( + interval_seconds=int(insp_raw.get("interval_seconds", 60)), + log_level=str(insp_raw.get("log_level", "INFO")).upper(), + dry_run=bool(insp_raw.get("dry_run", False)), + log_dir=str(insp_raw.get("log_dir", "/app/logs")), + log_filename=str(insp_raw.get("log_filename", "inspector.log")), + log_retention_days=int(insp_raw.get("log_retention_days", 30)), + heartbeat=bool(insp_raw.get("heartbeat", False)), + ) + + gotify_raw = ((raw.get("notifications") or {}).get("gotify")) or {} + gotify = GotifyConfig( + enabled=bool(gotify_raw.get("enabled", False)), + url=str(gotify_raw.get("url", "") or "").rstrip("/"), + token=str(gotify_raw.get("token", "") or ""), + priority=int(gotify_raw.get("priority", 5)), + ) + + pushover_raw = ((raw.get("notifications") or {}).get("pushover")) or {} + pushover = PushoverConfig( + enabled=bool(pushover_raw.get("enabled", False)), + token=str(pushover_raw.get("token", "") or ""), + user_key=str(pushover_raw.get("user_key", "") or ""), + priority=int(pushover_raw.get("priority", 5)), + device=str(pushover_raw.get("device", "") or ""), + sound=str(pushover_raw.get("sound", "") or ""), + ) + if pushover.enabled and (not pushover.token or not pushover.user_key): + raise ValueError( + "config: notifications.pushover.enabled is true but token and/or user_key is empty" + ) + + db_raw = raw.get("database") or {} + database = DatabaseConfig( + enabled=bool(db_raw.get("enabled", False)), + path=str(db_raw.get("path", "/app/data/events.db")), + retention_days=int(db_raw.get("retention_days", 30)), + ) + + hs_raw = raw.get("health_summary") or {} + health_summary = HealthSummaryConfig( + enabled=bool(hs_raw.get("enabled", False)), + interval_hours=int(hs_raw.get("interval_hours", 24)), + priority=int(hs_raw.get("priority", 3)), + ) + + maint_raw = raw.get("maintenance") or {} + maintenance = MaintenanceConfig( + enabled=bool(maint_raw.get("enabled", False)), + start=str(maint_raw.get("start", "00:00")), + end=str(maint_raw.get("end", "06:00")), + dir=str(maint_raw.get("dir", "/app/data/maintenance")), + max_hours=int(maint_raw.get("max_hours", 8)), + ) + + notif_mute_raw = raw.get("notification_mute") or {} + notification_mute = NotificationMuteConfig( + enabled=bool(notif_mute_raw.get("enabled", False)), + dir=str(notif_mute_raw.get("dir", "/app/data/muted")), + default_hours=int(notif_mute_raw.get("default_hours", 12)), + ) + + dash_raw = raw.get("dashboard") or {} + dashboard = DashboardConfig( + enabled=bool(dash_raw.get("enabled", False)), + host=str(dash_raw.get("host", "0.0.0.0")), + port=int(dash_raw.get("port", 8080)), + username=str(dash_raw.get("username", "admin")), + password=str(dash_raw.get("password", "")), + ) + + integrations_raw = raw.get("integrations") or {} + + uk_raw = integrations_raw.get("uptime_kuma") or {} + uptime_kuma = UptimeKumaConfig( + enabled=bool(uk_raw.get("enabled", False)), + url=str(uk_raw.get("url", "") or "").rstrip("/"), + status_page_slug=str(uk_raw.get("status_page_slug", "") or ""), + ) + + portainer_raw = integrations_raw.get("portainer") or {} + portainer_envs_raw = portainer_raw.get("environments") or {} + if not isinstance(portainer_envs_raw, dict): + raise ValueError( + "config: 'integrations.portainer.environments' must be a mapping of name -> settings" + ) + portainer_environments: dict[str, PortainerEnvironmentConfig] = {} + for env_name, env_settings in portainer_envs_raw.items(): + env_settings = env_settings or {} + portainer_environments[str(env_name)] = PortainerEnvironmentConfig( + base_url=str(env_settings.get("base_url", "") or "").rstrip("/"), + api_token_env=str(env_settings.get("api_token_env", "") or ""), + api_token=str(env_settings.get("api_token", "") or ""), + endpoint_id=int(env_settings.get("endpoint_id", 0)), + verify_ssl=bool(env_settings.get("verify_ssl", True)), + ) + + portainer = PortainerConfig( + enabled=bool(portainer_raw.get("enabled", False)), + default_environment=str(portainer_raw.get("default_environment", "") or ""), + stack_redeploy_cooldown_seconds=int(portainer_raw.get("stack_redeploy_cooldown_seconds", 300)), + post_redeploy_wait_seconds=int(portainer_raw.get("post_redeploy_wait_seconds", 90)), + environments=portainer_environments, + ) + + integrations = IntegrationsConfig(uptime_kuma=uptime_kuma, portainer=portainer) + + containers_raw = raw.get("containers") or {} + if not isinstance(containers_raw, dict): + raise ValueError("config: 'containers' must be a mapping of name -> settings") + + containers: list[ContainerConfig] = [] + for name, settings in containers_raw.items(): + settings = settings or {} + restart_on_raw = settings.get("restart_on") + if restart_on_raw is None: + restart_on_raw = ["UNHEALTHY", "NOT RUNNING"] + restart_on = frozenset(str(s).upper() for s in restart_on_raw) + unknown = restart_on - VALID_STATUSES + if unknown: + raise ValueError( + f"config: container {name!r} has unknown restart_on values: {sorted(unknown)}" + ) + http_check_raw = settings.get("http_check") or {} + http_check = HttpCheckConfig( + enabled=bool(http_check_raw.get("enabled", False)), + url=str(http_check_raw.get("url", "") or ""), + timeout_seconds=float(http_check_raw.get("timeout_seconds", 5.0)), + expected_status=int(http_check_raw.get("expected_status", 200)), + ) + + uptime_kuma_raw = settings.get("uptime_kuma") or {} + uptime_kuma = UptimeKumaMonitorConfig( + enabled=bool(uptime_kuma_raw.get("enabled", False)), + monitor_name=str(uptime_kuma_raw.get("monitor_name", "") or ""), + ) + + depends_on = str(settings.get("depends_on", "") or "").strip() + vpn_routed = bool(settings.get("vpn_routed", False)) + recovery_action = str(settings.get("recovery_action", "") or "").strip() + portainer_environment = str(settings.get("portainer_environment", "") or "").strip() + portainer_stack_name = str(settings.get("portainer_stack_name", "") or "").strip() + + if recovery_action and recovery_action not in VALID_RECOVERY_ACTIONS: + raise ValueError( + f"config: container {name!r} has unknown recovery_action " + f"{recovery_action!r}; expected one of {sorted(VALID_RECOVERY_ACTIONS)}" + ) + + if ( + recovery_action in ("portainer_stack_redeploy", "restart_then_redeploy") + and not portainer_stack_name + ): + raise ValueError( + f"config: container {name!r} has recovery_action " + f"{recovery_action!r} but no portainer_stack_name" + ) + + containers.append( + ContainerConfig( + name=str(name), + enabled=bool(settings.get("enabled", True)), + max_error_count=int(settings.get("max_error_count", 5)), + cooldown_seconds=int(settings.get("cooldown_seconds", 300)), + grace_period_seconds=int(settings.get("grace_period_seconds", 120)), + max_restarts_per_hour=int(settings.get("max_restarts_per_hour", 3)), + restart_on=restart_on, + maintenance_window=bool(settings.get("maintenance_window", False)), + color=str(settings.get("color", "") or "").strip(), + http_check=http_check, + uptime_kuma=uptime_kuma, + depends_on=depends_on, + vpn_routed=vpn_routed, + recovery_action=recovery_action, + portainer_environment=portainer_environment, + portainer_stack_name=portainer_stack_name, + ) + ) + + container_names = {c.name for c in containers} + for c in containers: + if c.depends_on and c.depends_on not in container_names: + raise ValueError( + f"config: container {c.name!r} has depends_on={c.depends_on!r}, " + f"which is not a configured container" + ) + + return AppConfig( + inspector=inspector, + gotify=gotify, + pushover=pushover, + database=database, + health_summary=health_summary, + maintenance=maintenance, + notification_mute=notification_mute, + dashboard=dashboard, + integrations=integrations, + containers=tuple(containers), + ) + + +# --- Roundtrip load/write (preserves comments and key order) ---------------- + +_yaml_rt = YAML(typ="rt") +_yaml_rt.preserve_quotes = True +_yaml_rt.indent(mapping=2, sequence=4, offset=2) +_yaml_rt.width = 4096 # don't line-wrap scalar values + + +def load_raw(path: str | Path): + """Load config.yml as a ruamel CommentedMap preserving comments.""" + with open(path, "r", encoding="utf-8") as fh: + data = _yaml_rt.load(fh) + return data if data is not None else {} + + +def dump_raw(raw) -> str: + """Serialize a raw (ruamel) config dict back to YAML text.""" + buf = io.StringIO() + _yaml_rt.dump(raw, buf) + return buf.getvalue() + + +def write_config_raw(path: str | Path, raw) -> None: + """Write `raw` back to `path`, keeping a `.bak` of the prior file. + + Prefers an atomic tempfile + os.replace() swap. Falls back to an + in-place write when the target is a single-file bind mount (as in + the Inspector Docker container, where /app/config.yml is a mount + point and can't be replaced from inside the container). The + fallback still writes the full text at once (not incrementally), + so a reader can see the old-or-new content, never partial lines + from this writer. + """ + path = Path(path) + text = dump_raw(raw) + tmp_path = path.with_suffix(path.suffix + ".tmp") + bak_path = path.with_suffix(path.suffix + ".bak") + + if path.exists(): + try: + shutil.copy2(path, bak_path) + except OSError: + pass # non-fatal; we still want to try the write + + def _fsync(fh) -> None: + try: + os.fsync(fh.fileno()) + except OSError: + pass + + try: + with open(tmp_path, "w", encoding="utf-8") as fh: + fh.write(text) + fh.flush() + _fsync(fh) + os.replace(tmp_path, path) + except OSError: + # Cross-device link (bind-mounted single file) or similar. + if tmp_path.exists(): + try: + tmp_path.unlink() + except OSError: + pass + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + fh.flush() + _fsync(fh) + + +class ConfigRef: + """Thread-safe holder for the live AppConfig and its raw ruamel dict. + + Writers call `reload_from_disk()` or `swap(new_raw, new_app)` under the + lock. Readers call `.get()` to grab the current AppConfig atomically. + """ + + def __init__(self, path: str | Path, app: AppConfig, raw) -> None: + self._path = Path(path) + self._app = app + self._raw = raw + self._lock = threading.Lock() + + @property + def path(self) -> Path: + return self._path + + def get(self) -> AppConfig: + with self._lock: + return self._app + + def get_raw(self): + """Return a deep copy of the current raw dict for safe mutation.""" + import copy + with self._lock: + return copy.deepcopy(self._raw) + + def swap(self, new_raw, new_app: AppConfig) -> None: + with self._lock: + self._raw = new_raw + self._app = new_app + + def reload_from_disk(self) -> AppConfig: + """Re-read config.yml from disk and swap in the new AppConfig.""" + with self._lock: + raw = load_raw(self._path) + app = build_config_from_raw(raw) + self._raw = raw + self._app = app + return app diff --git a/src/inspector/dashboard.py b/src/inspector/dashboard.py new file mode 100755 index 0000000..7edd4e4 --- /dev/null +++ b/src/inspector/dashboard.py @@ -0,0 +1,1601 @@ +from __future__ import annotations + +import json +import logging +import re +import secrets +import sqlite3 +import threading +import time +from base64 import b64decode +from datetime import date, datetime, timedelta, timezone +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib import resources +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +from . import __version__ +from .config import ( + AppConfig, + ConfigRef, + ContainerConfig, + build_config_from_raw, + write_config_raw, +) +from .formatting import PACIFIC, format_timestamp +from .health import CONTAINER_NOT_FOUND, ERROR, HEALTHY, NOT_RUNNING, UNHEALTHY, UNREACHABLE +from .maintenance import MaintenanceTracker +from .notification_mute import NotificationMuteTracker +from .state import ContainerState, utcnow + + +log = logging.getLogger("inspector.dashboard") + +_SESSION_MAX_AGE = 86400 # seconds a login cookie stays valid +_UNCHANGED_SECRET_PLACEHOLDER = "__UNCHANGED__" # UI sentinel: keep existing secret value + +# Config keys that require an Inspector container restart to take effect. +# Everything else edited through /settings hot-applies on the next loop +# iteration via ConfigRef. +_RESTART_REQUIRED_SECTIONS = { + "inspector_logging", # log_dir/log_filename/log_level/log_retention_days + "database", + "health_summary", + "maintenance", + "notification_mute", + "dashboard", # host/port bound at startup; username/password hot-apply + "integrations", +} + +# 16-color palette (Tailwind 400-level), evenly spread around the hue wheel. +# Used for auto-assigning container identity colors; wraps at 16. +_CONTAINER_PALETTE = [ + "#60a5fa", # blue + "#34d399", # emerald + "#f472b6", # pink + "#fb923c", # orange + "#a78bfa", # violet + "#2dd4bf", # teal + "#fbbf24", # amber + "#818cf8", # indigo + "#e879f9", # fuchsia + "#22d3ee", # cyan + "#4ade80", # green + "#c084fc", # purple + "#38bdf8", # sky + "#fb7185", # rose + "#a3e635", # lime + "#facc15", # yellow +] + +_HEX_COLOR_RE = re.compile(r"^#[0-9a-fA-F]{3}([0-9a-fA-F]{3})?$") + + +def _resolve_color(cfg: ContainerConfig, index: int) -> str: + c = cfg.color.strip() + return c if _HEX_COLOR_RE.match(c) else _CONTAINER_PALETTE[index % len(_CONTAINER_PALETTE)] + + +def _color_map( + states: dict[str, ContainerState], + containers: dict[str, ContainerConfig], +) -> dict[str, str]: + return {name: _resolve_color(containers[name], i) for i, name in enumerate(states)} + + +def _compute_phase( + state: ContainerState, + cfg: ContainerConfig, + maintenance_active: bool, + now: datetime, +) -> tuple[str, datetime | None]: + """Return (phase_name, phase_until) for the current Inspector logic state. + + phase_until is the UTC datetime when this phase is expected to end, or None + when the phase has no fixed end time (maintenance, observe_only, ok, failing). + """ + if maintenance_active: + return "maintenance", None + if state.in_cooldown(now): + return "cooldown", state.cooldown_until + if state.in_grace(now, cfg.grace_period_seconds) and state.last_restart_at is not None: + grace_until = state.last_restart_at + timedelta(seconds=cfg.grace_period_seconds) + return "grace", grace_until + if state.restarts_last_hour(now) >= cfg.max_restarts_per_hour: + hour_ago = now - timedelta(seconds=3600) + recent = sorted(t for t in state.restart_history if t >= hour_ago) + limit_until = (min(recent) + timedelta(seconds=3600)) if recent else None + return "restart_limit", limit_until + if not cfg.restart_on: + return "observe_only", None + if state.error_count > 0: + return "failing", None + return "ok", None + + +def _status_snapshot( + states: dict[str, ContainerState], + containers: dict[str, ContainerConfig], + maintenance: MaintenanceTracker, + notification_mute: NotificationMuteTracker, + interval_seconds: int = 60, +) -> list[dict]: + now = utcnow() + rows = [] + for i, (name, state) in enumerate(states.items()): + cfg = containers[name] + cooldown_until = state.cooldown_until + last_restart_at = state.last_restart_at + muted_until = notification_mute.muted_until(name) + maintenance_active = maintenance.peek(name) + phase, phase_until = _compute_phase(state, cfg, maintenance_active, now) + window = maintenance.next_scheduled_window(name) + if window is not None: + window_start = window[0].astimezone(timezone.utc).isoformat() + window_end = window[1].astimezone(timezone.utc).isoformat() + else: + window_start = window_end = None + rows.append( + { + "name": name, + "current_status": state.current_status, + "error_count": state.error_count, + "max_error_count": cfg.max_error_count, + "cooldown_until": cooldown_until.isoformat() if cooldown_until else None, + "last_restart_at": last_restart_at.isoformat() if last_restart_at else None, + "maintenance_active": maintenance_active, + "maintenance_scheduled": window is not None, + "maintenance_window_start": window_start, + "maintenance_window_end": window_end, + "notification_muted": muted_until is not None, + "notification_muted_until": muted_until.isoformat() if muted_until else None, + "color": _resolve_color(cfg, i), + "phase": phase, + "phase_until": phase_until.isoformat() if phase_until else None, + "last_check_at": state.last_check_at.isoformat() if state.last_check_at else None, + "interval_seconds": interval_seconds, + } + ) + return rows + + +def _parse_date_param(value: str | None) -> date: + """Parse a ?date=YYYY-MM-DD query param; falls back to "today" in + America/Los_Angeles on missing/invalid input.""" + if value: + try: + return date.fromisoformat(value) + except ValueError: + pass + return datetime.now(PACIFIC).date() + + +def _day_bounds_utc(day: date) -> tuple[datetime, datetime]: + """UTC [start, end) bounds for a Pacific calendar day, extended one hour + into the previous day for the lookback-overlap.""" + midnight = datetime.combine(day, datetime.min.time(), tzinfo=PACIFIC) + start = (midnight - timedelta(hours=1)).astimezone(timezone.utc) + end = (midnight + timedelta(days=1)).astimezone(timezone.utc) + return start, end + + +def _read_events_range(db_path: str, start: datetime, end: datetime) -> list[dict]: + try: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + try: + cur = conn.execute( + "SELECT ts, container, event_type, old_status, new_status, error_count, message " + "FROM events WHERE ts >= ? AND ts < ? ORDER BY ts DESC", + (start.isoformat(), end.isoformat()), + ) + columns = [c[0] for c in cur.description] + return [dict(zip(columns, row)) for row in cur.fetchall()] + finally: + conn.close() + except Exception as exc: + log.debug("dashboard: event history unavailable: %s", exc) + return [] + + +def _hour_markers(day: date) -> list[datetime]: + """24 Pacific-local hour boundaries for `day`, 00:00..23:00 ascending.""" + midnight = datetime.combine(day, datetime.min.time(), tzinfo=PACIFIC) + return [midnight + timedelta(hours=h) for h in range(24)] + + +def _group_events_by_hour( + events: list[dict], day: date +) -> tuple[list[dict], dict[datetime, list[dict]]]: + """Split `events` (ts as UTC ISO strings) into: + - buckets: each of the 24 hour-marker datetimes (Pacific) -> events + whose Pacific-local hour matches (newest-first, preserving input order) + - overlap: events from the previous day's 11pm-midnight Pacific hour + (the 1hr lookback window) + """ + markers = _hour_markers(day) + buckets: dict[datetime, list[dict]] = {m: [] for m in markers} + overlap: list[dict] = [] + prev_hour_start = markers[0] - timedelta(hours=1) + + for ev in events: + try: + ts = datetime.fromisoformat(ev["ts"]).astimezone(PACIFIC) + except (ValueError, KeyError): + continue + if prev_hour_start <= ts < markers[0]: + overlap.append(ev) + continue + hour_start = ts.replace(minute=0, second=0, microsecond=0) + if hour_start in buckets: + buckets[hour_start].append(ev) + + return overlap, buckets + + +def _escape(value: object) -> str: + text = "" if value is None else str(value) + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +_STATUS_BADGE_CLASS = { + HEALTHY: "healthy", + UNHEALTHY: "unhealthy", + NOT_RUNNING: "unhealthy", + CONTAINER_NOT_FOUND: "unhealthy", + ERROR: "unhealthy", + UNREACHABLE: "unhealthy", +} + + +def _status_badge_class(status: str | None) -> str: + return _STATUS_BADGE_CLASS.get(status or "", "muted") + + +_EVENT_DOT_CLASS = { + "restart": "restart", + "restart_dry_run": "restart", + "restart_failed": "failure", + "restart_limit": "failure", + "failure": "failure", + "not_found": "failure", + "recovery": "recovery", + "maintenance_enter": "maintenance", + "maintenance_exit": "maintenance", +} + + +def _event_dot_class(event_type: str) -> str: + return _EVENT_DOT_CLASS.get(event_type, "") + + +def _load_asset(name: str) -> bytes: + try: + return (resources.files(__package__) / "assets" / name).read_bytes() + except Exception as exc: + log.debug("dashboard: could not load asset %r: %s", name, exc) + return b"" + + +_ICON_PNG = _load_asset("inspector_icon_outline.png") + +# Filename -> Content-Type, for files in src/inspector/assets/ served at /static/. +_STATIC_FILES = { + "icon.png": "image/png", + "favicon.ico": "image/x-icon", + "favicon.svg": "image/svg+xml", + "favicon-96x96.png": "image/png", + "apple-touch-icon.png": "image/png", + "web-app-manifest-192x192.png": "image/png", + "web-app-manifest-512x512.png": "image/png", + "site.webmanifest": "application/manifest+json", +} +_STATIC_ASSETS = { + "icon.png": _ICON_PNG, + **{name: _load_asset(name) for name in _STATIC_FILES if name != "icon.png"}, +} + + +_STYLE = """""" + + +def _page_head(refresh: bool, active: str) -> str: + refresh_meta = '\n' if refresh else "" + status_class = "active" if active == "status" else "" + events_class = "active" if active == "events" else "" + settings_class = "active" if active == "settings" else "" + meta_text = f"v{__version__}" + if refresh: + meta_text += " · auto-refreshes every 30s" + return ( + "\n" + '\n\n' + '\n' + '\n' + "Inspector\n" + '\n' + '\n' + '\n' + '\n' + '\n' + + refresh_meta + + '\n' + '\n' + '\n' + + _STYLE + + "\n\n\n" + "
\n" + '
\n' + ' Inspector\n' + "
\n" + '
Inspector
\n' + '
Monitor. Restart. Report.
\n' + "
\n" + "
\n" + '
\n' + f' Status\n' + f' Events\n' + f' Settings\n' + "
\n" + f'
{meta_text}Sign out
\n' + "
\n" + "
\n" + ) + + +_PAGE_TAIL = ( + "
\n" + f"
Inspector v{__version__} — Monitor. Restart. Report.
\n" + "\n" + "\n\n" +) + + +def _render_status_page( + rows: list[dict], + maintenance_enabled: bool, + notification_mute_enabled: bool, + csrf_token: str, +) -> bytes: + healthy = sum(1 for r in rows if _status_badge_class(r["current_status"]) == "healthy") + issues = sum(1 for r in rows if _status_badge_class(r["current_status"]) == "unhealthy") + in_maintenance = sum(1 for r in rows if r["maintenance_active"]) + + parts = [_page_head(refresh=True, active="status")] + parts.append('
\n') + parts.append( + f'
' + f'
{len(rows)}
Containers
\n' + ) + parts.append( + f'
' + f'
{healthy}
Healthy
\n' + ) + parts.append( + f'
' + f'
{issues}
Issues
\n' + ) + parts.append( + f'
' + f'
{in_maintenance}
In maintenance
\n' + ) + parts.append("
\n") + + parts.append('

Containers

\n') + parts.append('
\n') + for row in rows: + name = _escape(row["name"]) + status = row["current_status"] or "UNKNOWN" + max_error_count = row["max_error_count"] + error_count = row["error_count"] + if error_count == 0: + pct, bar_class = 100, "ok" + else: + pct = min(100, round(error_count / max_error_count * 100)) if max_error_count > 0 else 100 + bar_class = "warn" + cooldown = _escape(format_timestamp(row["cooldown_until"])) + last_restart = _escape(format_timestamp(row["last_restart_at"])) or "Never" + + col = _escape(row["color"]) + parts.append(f'
\n') + parts.append('
') + parts.append(f'{name}') + parts.append('') + parts.append(f'{_escape(status)}') + if row["maintenance_active"]: + parts.append('Maintenance') + if row["notification_muted"]: + parts.append('🔇 Muted') + parts.append("") + parts.append("
\n") + parts.append( + f'
Errors' + f'
' + f"{error_count}/{max_error_count}
\n" + ) + + # Phase row with live countdown timer + phase = row.get("phase", "ok") + phase_until = row.get("phase_until") + last_check_at = row.get("last_check_at") + interval_s = row.get("interval_seconds", 60) + if phase == "cooldown" and phase_until: + parts.append( + f'
' + f'⏱ Cooldown — remaining' + f"
\n" + ) + elif phase == "grace" and phase_until: + parts.append( + f'
' + f'⏱ Grace period — remaining' + f"
\n" + ) + elif phase == "restart_limit": + if phase_until: + parts.append( + f'
' + f'⚠ Restart limit — resets in ' + f"
\n" + ) + else: + parts.append('
⚠ Restart limit reached
\n') + elif phase == "failing": + if last_check_at: + try: + next_ts = (datetime.fromisoformat(last_check_at) + timedelta(seconds=interval_s)).isoformat() + parts.append( + f'
' + f'⚠ Failing {error_count}/{max_error_count} — next check ' + f'' + f"
\n" + ) + except ValueError: + parts.append(f'
⚠ Failing {error_count}/{max_error_count}
\n') + else: + parts.append(f'
⚠ Failing {error_count}/{max_error_count}
\n') + elif phase == "observe_only": + parts.append('
👁 Observe only
\n') + elif phase == "maintenance": + window_start = row.get("maintenance_window_start") + window_end = row.get("maintenance_window_end") + in_scheduled_window = False + if window_start and window_end: + try: + now_dt = utcnow() + in_scheduled_window = ( + datetime.fromisoformat(window_start) <= now_dt < datetime.fromisoformat(window_end) + ) + except ValueError: + in_scheduled_window = False + if in_scheduled_window: + parts.append( + f'
' + f'🔧 In maintenance — ends in ' + f"
\n" + ) + else: + parts.append('
🔧 In maintenance
\n') + else: # "ok" + if last_check_at: + try: + next_ts = (datetime.fromisoformat(last_check_at) + timedelta(seconds=interval_s)).isoformat() + parts.append( + f'
' + f'Next check ' + f"
\n" + ) + except ValueError: + pass + + parts.append('
') + if cooldown: + parts.append(f"
Cooldown until: {cooldown}
") + parts.append(f"
Last restart: {last_restart}
") + if row["notification_muted"]: + muted_until = _escape(format_timestamp(row["notification_muted_until"])) + parts.append(f"
Notifications muted until: {muted_until}
") + if row.get("maintenance_scheduled") and not row["maintenance_active"]: + win_start = row.get("maintenance_window_start") + win_end = row.get("maintenance_window_end") + if win_start and win_end: + start_disp = _escape(format_timestamp(win_start)) + end_disp = _escape(format_timestamp(win_end)) + parts.append( + f'
🔧 Next maintenance: in ' + f' ' + f"({start_disp} – {end_disp})
" + ) + parts.append("
\n") + + action_buttons: list[str] = [] + csrf_field = f'' + if maintenance_enabled: + action = "exit" if row["maintenance_active"] else "enter" + label = "Exit maintenance" if row["maintenance_active"] else "Enter maintenance" + btn_class = "action exit" if row["maintenance_active"] else "action" + action_buttons.append( + f'
{csrf_field}' + f'
' + ) + if notification_mute_enabled: + action = "unmute" if row["notification_muted"] else "mute" + label = "Unmute notifications" if row["notification_muted"] else "Mute notifications" + btn_class = "action exit" if row["notification_muted"] else "action" + action_buttons.append( + f'
{csrf_field}' + f'
' + ) + if action_buttons: + parts.append('
' + "".join(action_buttons) + "
\n") + parts.append("
\n") + parts.append("
\n") + parts.append(_PAGE_TAIL) + return "".join(parts).encode("utf-8") + + +def _render_events_page( + events: list[dict], + database_enabled: bool, + selected_day: date, + container_colors: dict[str, str] | None = None, +) -> bytes: + parts = [_page_head(refresh=False, active="events")] + + prev_day = (selected_day - timedelta(days=1)).isoformat() + next_day = (selected_day + timedelta(days=1)).isoformat() + today = datetime.now(PACIFIC).date() + day_label = selected_day.strftime("%A, %B %-d, %Y") + + parts.append('
\n') + parts.append(f'← Prev day\n') + parts.append(f'{_escape(day_label)}\n') + if selected_day >= today: + parts.append('Next day →\n') + else: + parts.append(f'Next day →\n') + parts.append("
\n") + + parts.append('

Events

\n') + + if not database_enabled: + parts.append( + '

Event history is not enabled. To enable it, add' + " database:
  enabled: true
to" + " config.yml and restart Inspector.

\n" + ) + parts.append(_PAGE_TAIL) + return "".join(parts).encode("utf-8") + + overlap, buckets = _group_events_by_hour(events, selected_day) + + parts.append('
\n') + for marker in reversed(_hour_markers(selected_day)): + label = marker.strftime("%-I %p") # "12 AM" .. "11 PM" + parts.append( + '
' + f"{_escape(label)}
\n" + ) + for ev in buckets[marker]: + parts.append(_render_event_item(ev, container_colors)) + + if overlap: + parts.append( + '
' + "11 PM (yesterday)
\n" + ) + for ev in overlap: + parts.append(_render_event_item(ev, container_colors)) + + parts.append("
\n") + parts.append(_PAGE_TAIL) + return "".join(parts).encode("utf-8") + + +def _render_event_item(ev: dict, container_colors: dict[str, str] | None) -> str: + old = _escape(ev.get("old_status")) or "—" + new = _escape(ev.get("new_status")) or "—" + message = _escape(ev.get("message")) + if ev.get("old_status") or ev.get("new_status"): + msg = f"{old} → {new} — {message}" + else: + msg = message + col = (container_colors or {}).get(ev["container"], "") + item_style = f' style="--col:{_escape(col)}"' if col else "" + con_class = "tl-container tl-chip" if col else "tl-container" + return ( + f'
' + f'' + '
' + f'{_escape(format_timestamp(ev["ts"]))}' + f'{_escape(ev["container"])}' + f'{_escape(ev["event_type"])}' + f"
{msg}
\n" + ) + + +def _render_login_page(error: bool = False) -> bytes: + error_html = ( + '\n' if error else "" + ) + head = ( + "\n" + '\n\n' + '\n' + '\n' + "Inspector — Sign in\n" + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + '\n' + + _STYLE + + "\n\n\n" + '\n" + "\n\n" + ) + return head.encode("utf-8") + + +# --- Settings page ---------------------------------------------------------- + +from urllib.parse import quote as _url_quote # noqa: E402 + + +class _SettingsError(ValueError): + pass + + +def _form_str(form: dict, key: str, default: str = "") -> str: + return form.get(key, [default])[0] + + +def _form_bool(form: dict, key: str) -> bool: + return _form_str(form, key, "").lower() in ("1", "true", "on", "yes") + + +def _form_int(form: dict, key: str, default: int = 0) -> int: + try: + return int(_form_str(form, key, str(default))) + except (TypeError, ValueError): + return default + + +def _apply_secret(form: dict, raw_section: dict, form_key: str, yaml_key: str) -> None: + """Store form value under raw_section[yaml_key] unless it's the mask placeholder.""" + submitted = _form_str(form, form_key, "") + if submitted == _UNCHANGED_SECRET_PLACEHOLDER: + return + raw_section[yaml_key] = submitted + + +def _apply_settings_update(config_ref: ConfigRef, section: str, form: dict) -> None: + """Validate the form, merge into the raw config, write atomically, swap ref.""" + raw = config_ref.get_raw() + + if section == "inspector": + insp = raw.setdefault("inspector", {}) + insp["interval_seconds"] = _form_int(form, "interval_seconds", 60) + insp["log_level"] = _form_str(form, "log_level", "INFO").upper() + insp["dry_run"] = _form_bool(form, "dry_run") + insp["heartbeat"] = _form_bool(form, "heartbeat") + insp["log_retention_days"] = _form_int(form, "log_retention_days", 30) + elif section == "gotify": + notif = raw.setdefault("notifications", {}) + gotify = notif.setdefault("gotify", {}) + gotify["enabled"] = _form_bool(form, "enabled") + gotify["url"] = _form_str(form, "url", "") + _apply_secret(form, gotify, "token", "token") + gotify["priority"] = _form_int(form, "priority", 5) + elif section == "pushover": + notif = raw.setdefault("notifications", {}) + pushover = notif.setdefault("pushover", {}) + pushover["enabled"] = _form_bool(form, "enabled") + _apply_secret(form, pushover, "token", "token") + _apply_secret(form, pushover, "user_key", "user_key") + pushover["priority"] = _form_int(form, "priority", 5) + pushover["device"] = _form_str(form, "device", "") + pushover["sound"] = _form_str(form, "sound", "") + elif section == "dashboard": + dash = raw.setdefault("dashboard", {}) + dash["enabled"] = _form_bool(form, "enabled") + dash["username"] = _form_str(form, "username", "admin") + _apply_secret(form, dash, "password", "password") + elif section == "database": + db = raw.setdefault("database", {}) + db["enabled"] = _form_bool(form, "enabled") + db["retention_days"] = _form_int(form, "retention_days", 30) + elif section == "health_summary": + hs = raw.setdefault("health_summary", {}) + hs["enabled"] = _form_bool(form, "enabled") + hs["interval_hours"] = _form_int(form, "interval_hours", 24) + hs["priority"] = _form_int(form, "priority", 3) + elif section == "maintenance": + m = raw.setdefault("maintenance", {}) + m["enabled"] = _form_bool(form, "enabled") + m["start"] = _form_str(form, "start", "00:00") + m["end"] = _form_str(form, "end", "06:00") + m["max_hours"] = _form_int(form, "max_hours", 8) + elif section == "notification_mute": + nm = raw.setdefault("notification_mute", {}) + nm["enabled"] = _form_bool(form, "enabled") + nm["default_hours"] = _form_int(form, "default_hours", 12) + elif section.startswith("container:"): + name = section.split(":", 1)[1] + containers = raw.get("containers") or {} + if name not in containers: + raise _SettingsError(f"unknown container {name!r}") + cfg = containers[name] or {} + cfg["enabled"] = _form_bool(form, "enabled") + cfg["max_error_count"] = _form_int(form, "max_error_count", 5) + cfg["cooldown_seconds"] = _form_int(form, "cooldown_seconds", 300) + cfg["grace_period_seconds"] = _form_int(form, "grace_period_seconds", 120) + cfg["max_restarts_per_hour"] = _form_int(form, "max_restarts_per_hour", 3) + cfg["color"] = _form_str(form, "color", "") + restart_on = form.get("restart_on", []) + if restart_on: + cfg["restart_on"] = [s.upper() for s in restart_on] + containers[name] = cfg + raw["containers"] = containers + else: + raise _SettingsError(f"unknown settings section {section!r}") + + # Validate before writing. + try: + new_app = build_config_from_raw(raw) + except Exception as exc: + raise _SettingsError(str(exc)) from exc + + try: + write_config_raw(config_ref.path, raw) + except OSError as exc: + raise _SettingsError(f"could not write config.yml: {exc}") from exc + + config_ref.swap(raw, new_app) + log.info("Settings saved: section=%s", section) + + +_RESTART_LABEL = ( + '' + "requires restart" +) + + +def _settings_field( + label: str, + name: str, + value: str, + *, + kind: str = "text", + hint: str = "", + requires_restart: bool = False, +) -> str: + label_html = _escape(label) + if requires_restart: + label_html += " " + _RESTART_LABEL + hint_html = f'
{_escape(hint)}
' if hint else "" + if kind == "checkbox": + checked = "checked" if value in ("1", "True", "true", "on") else "" + return ( + f'
' + f'{hint_html}
" + ) + return ( + f'
' + f'' + f'{hint_html}
' + ) + + +def _settings_secret_field(label: str, name: str, has_value: bool) -> str: + hint = "leave as-is to keep the current value; empty to clear" + placeholder = _UNCHANGED_SECRET_PLACEHOLDER if has_value else "" + return ( + f'
' + f'' + f'' + f'
{_escape(hint)}
' + ) + + +def _render_settings_page(app: AppConfig, csrf_token: str, flash: str) -> bytes: + def _form(section: str, title: str, body: str) -> str: + return ( + f'

{_escape(title)}

' + f'
' + f'' + f'{body}' + f'' + f"
\n" + ) + + parts = [_page_head(refresh=False, active="settings")] + + if flash: + if flash.startswith("ok:"): + parts.append( + f'
Saved: {_escape(flash[3:])}. ' + "Fields marked requires restart take effect after Inspector restarts.
\n" + ) + elif flash.startswith("error:"): + parts.append(f'
{_escape(flash[6:])}
\n') + + parts.append('

Settings

\n') + parts.append('
\n') + + # Inspector + insp = app.inspector + body = ( + _settings_field("interval_seconds", "interval_seconds", str(insp.interval_seconds), kind="number") + + _settings_field("log_level", "log_level", insp.log_level, hint="DEBUG, INFO, WARNING, ERROR") + + _settings_field("dry_run", "dry_run", "on" if insp.dry_run else "", kind="checkbox") + + _settings_field("heartbeat", "heartbeat", "on" if insp.heartbeat else "", kind="checkbox") + + _settings_field("log_retention_days", "log_retention_days", str(insp.log_retention_days), kind="number", requires_restart=True) + ) + parts.append(_form("inspector", "Inspector", body)) + + # Gotify + g = app.gotify + body = ( + _settings_field("enabled", "enabled", "on" if g.enabled else "", kind="checkbox", requires_restart=True) + + _settings_field("url", "url", g.url, requires_restart=True) + + _settings_secret_field("token", "token", bool(g.token)) + + _settings_field("priority", "priority", str(g.priority), kind="number") + ) + parts.append(_form("gotify", "Gotify notifications", body)) + + # Pushover + p = app.pushover + body = ( + _settings_field("enabled", "enabled", "on" if p.enabled else "", kind="checkbox", requires_restart=True) + + _settings_secret_field("token", "token", bool(p.token)) + + _settings_secret_field("user_key", "user_key", bool(p.user_key)) + + _settings_field("priority", "priority", str(p.priority), kind="number") + + _settings_field("device", "device", p.device, hint="optional; blank = all devices") + + _settings_field("sound", "sound", p.sound, hint="optional; blank = user default") + ) + parts.append(_form("pushover", "Pushover notifications", body)) + + # Dashboard + d = app.dashboard + body = ( + _settings_field("enabled", "enabled", "on" if d.enabled else "", kind="checkbox", requires_restart=True) + + _settings_field("username", "username", d.username) + + _settings_secret_field("password", "password", bool(d.password)) + ) + parts.append(_form("dashboard", "Dashboard", body)) + + # Database + db = app.database + body = ( + _settings_field("enabled", "enabled", "on" if db.enabled else "", kind="checkbox", requires_restart=True) + + _settings_field("retention_days", "retention_days", str(db.retention_days), kind="number", requires_restart=True) + ) + parts.append(_form("database", "Event history (SQLite)", body)) + + # Health summary + hs = app.health_summary + body = ( + _settings_field("enabled", "enabled", "on" if hs.enabled else "", kind="checkbox", requires_restart=True) + + _settings_field("interval_hours", "interval_hours", str(hs.interval_hours), kind="number", requires_restart=True) + + _settings_field("priority", "priority", str(hs.priority), kind="number") + ) + parts.append(_form("health_summary", "Health summary digest", body)) + + # Maintenance window + m = app.maintenance + body = ( + _settings_field("enabled", "enabled", "on" if m.enabled else "", kind="checkbox", requires_restart=True) + + _settings_field("start", "start", m.start, hint="HH:MM, 24h") + + _settings_field("end", "end", m.end, hint="HH:MM, 24h") + + _settings_field("max_hours", "max_hours", str(m.max_hours), kind="number") + ) + parts.append(_form("maintenance", "Scheduled maintenance window", body)) + + # Notification mute + nm = app.notification_mute + body = ( + _settings_field("enabled", "enabled", "on" if nm.enabled else "", kind="checkbox", requires_restart=True) + + _settings_field("default_hours", "default_hours", str(nm.default_hours), kind="number") + ) + parts.append(_form("notification_mute", "Notification mute", body)) + + # Containers (edit of existing only; add/remove requires YAML edit + restart) + for c in app.containers: + restart_on_opts = ["HEALTHY", "UNHEALTHY", "STARTING", "NO HEALTH CHECK", + "NOT RUNNING", "CONTAINER NOT FOUND", "ERROR", "UNKNOWN", "UNREACHABLE"] + checkboxes = [] + for s in restart_on_opts: + checked = "checked" if s in c.restart_on else "" + checkboxes.append( + f'' + ) + restart_on_html = ( + '
' + f'
{"".join(checkboxes)}
' + ) + body = ( + _settings_field("enabled", "enabled", "on" if c.enabled else "", kind="checkbox") + + _settings_field("max_error_count", "max_error_count", str(c.max_error_count), kind="number") + + _settings_field("cooldown_seconds", "cooldown_seconds", str(c.cooldown_seconds), kind="number") + + _settings_field("grace_period_seconds", "grace_period_seconds", str(c.grace_period_seconds), kind="number") + + _settings_field("max_restarts_per_hour", "max_restarts_per_hour", str(c.max_restarts_per_hour), kind="number") + + _settings_field("color", "color", c.color, hint="hex, e.g. #60a5fa; blank = auto") + + restart_on_html + ) + parts.append(_form(f"container:{c.name}", f"Container — {c.name}", body)) + + parts.append("
\n") + parts.append( + '

To add or remove a container, ' + "edit config.yml and restart Inspector.

\n" + ) + parts.append(_PAGE_TAIL) + return "".join(parts).encode("utf-8") + + +class _DashboardHTTPServer(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__( + self, + address: tuple[str, int], + handler_cls, + *, + config_ref: ConfigRef, + states: dict[str, ContainerState], + maintenance: MaintenanceTracker, + notification_mute: NotificationMuteTracker, + ): + super().__init__(address, handler_cls) + self.config_ref = config_ref + self.states = states + self.maintenance = maintenance + self.notification_mute = notification_mute + # token -> (expiry_timestamp, csrf_token) + self._sessions: dict[str, tuple[float, str]] = {} + + def current_app(self) -> AppConfig: + return self.config_ref.get() + + def current_containers(self) -> dict[str, ContainerConfig]: + return {c.name: c for c in self.config_ref.get().containers} + + +class _Handler(BaseHTTPRequestHandler): + server: _DashboardHTTPServer + + def log_message(self, fmt: str, *args) -> None: + log.debug("%s - " + fmt, self.address_string(), *args) + + def _session_token(self) -> str | None: + for part in self.headers.get("Cookie", "").split(";"): + k, _, v = part.strip().partition("=") + if k.strip() == "inspector_session": + return v.strip() + return None + + def _is_authenticated(self) -> bool: + # Session cookie + now = time.time() + token = self._session_token() + if token: + entry = self.server._sessions.get(token) + if entry and now < entry[0]: + return True + self.server._sessions.pop(token, None) + + # Basic Auth + header = self.headers.get("Authorization", "") + if header.startswith("Basic "): + try: + decoded = b64decode(header[len("Basic "):]).decode("utf-8") + username, _, password = decoded.partition(":") + except Exception: + username, password = "", "" + cfg = self.server.current_app().dashboard + if secrets.compare_digest(username, cfg.username) and secrets.compare_digest( + password, cfg.password + ): + return True + + return False + + def _csrf_token_for_session(self) -> str: + """Return the CSRF token bound to the current session, or '' if none.""" + token = self._session_token() + if not token: + return "" + entry = self.server._sessions.get(token) + return entry[1] if entry else "" + + def _check_csrf(self, submitted: str) -> bool: + expected = self._csrf_token_for_session() + if not expected or not submitted: + return False + return secrets.compare_digest(expected, submitted) + + def _check_auth(self) -> bool: + if self._is_authenticated(): + return True + + # Browsers get a redirect to the login page; API clients get a plain 401. + if "text/html" in self.headers.get("Accept", ""): + self.send_response(HTTPStatus.SEE_OTHER) + self.send_header("Location", "/login") + self.send_header("Content-Length", "0") + self.end_headers() + else: + self.send_response(HTTPStatus.UNAUTHORIZED) + self.send_header("WWW-Authenticate", 'Basic realm="Inspector"') + self.send_header("Content-Length", "0") + self.end_headers() + return False + + def _send_bytes(self, status: HTTPStatus, content_type: str, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + parsed = urlparse(self.path) + path = parsed.path + query = parse_qs(parsed.query) + server = self.server + + # Unprotected routes (served before auth check) + if path == "/login": + self._send_bytes(HTTPStatus.OK, "text/html; charset=utf-8", _render_login_page()) + return + + if path == "/logout": + for part in self.headers.get("Cookie", "").split(";"): + k, _, v = part.strip().partition("=") + if k.strip() == "inspector_session": + server._sessions.pop(v.strip(), None) + self.send_response(HTTPStatus.SEE_OTHER) + self.send_header("Location", "/login") + self.send_header("Set-Cookie", "inspector_session=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if path.startswith("/static/") and path[len("/static/"):] in _STATIC_FILES: + name = path[len("/static/"):] + data = _STATIC_ASSETS[name] + if data: + self._send_bytes(HTTPStatus.OK, _STATIC_FILES[name], data) + else: + self._send_bytes(HTTPStatus.NOT_FOUND, "text/plain; charset=utf-8", b"not found") + return + + if not self._check_auth(): + return + + app = server.current_app() + containers = server.current_containers() + interval_s = app.inspector.interval_seconds + + if path == "/": + rows = _status_snapshot(server.states, containers, server.maintenance, server.notification_mute, interval_s) + csrf = self._csrf_token_for_session() + body = _render_status_page(rows, app.maintenance.enabled, app.notification_mute.enabled, csrf) + self._send_bytes(HTTPStatus.OK, "text/html; charset=utf-8", body) + elif path == "/api/status": + rows = _status_snapshot(server.states, containers, server.maintenance, server.notification_mute, interval_s) + body = json.dumps(rows).encode("utf-8") + self._send_bytes(HTTPStatus.OK, "application/json", body) + elif path == "/events": + selected_day = _parse_date_param(query.get("date", [None])[0]) + colors = _color_map(server.states, containers) + if app.database.enabled: + start, end = _day_bounds_utc(selected_day) + events = _read_events_range(app.database.path, start, end) + else: + events = [] + body = _render_events_page(events, app.database.enabled, selected_day, colors) + self._send_bytes(HTTPStatus.OK, "text/html; charset=utf-8", body) + elif path == "/api/events": + selected_day = _parse_date_param(query.get("date", [None])[0]) + if app.database.enabled: + start, end = _day_bounds_utc(selected_day) + events = _read_events_range(app.database.path, start, end) + else: + events = [] + body = json.dumps(events).encode("utf-8") + self._send_bytes(HTTPStatus.OK, "application/json", body) + elif path == "/settings": + csrf = self._csrf_token_for_session() + flash = query.get("flash", [""])[0] + body = _render_settings_page(app, csrf, flash) + self._send_bytes(HTTPStatus.OK, "text/html; charset=utf-8", body) + else: + self._send_bytes(HTTPStatus.NOT_FOUND, "text/plain; charset=utf-8", b"not found") + + def do_POST(self) -> None: + path = urlparse(self.path).path + server = self.server + + if path == "/login": + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length) if length > 0 else b"" + data = parse_qs(raw.decode("utf-8", errors="replace")) + username = data.get("username", [""])[0] + password = data.get("password", [""])[0] + cfg = server.current_app().dashboard + if secrets.compare_digest(username, cfg.username) and secrets.compare_digest( + password, cfg.password + ): + token = secrets.token_urlsafe(32) + csrf = secrets.token_urlsafe(32) + server._sessions[token] = (time.time() + _SESSION_MAX_AGE, csrf) + self.send_response(HTTPStatus.SEE_OTHER) + self.send_header("Location", "/") + self.send_header( + "Set-Cookie", + f"inspector_session={token}; Path=/; HttpOnly; SameSite=Lax", + ) + self.send_header("Content-Length", "0") + self.end_headers() + else: + self._send_bytes( + HTTPStatus.UNAUTHORIZED, "text/html; charset=utf-8", _render_login_page(error=True) + ) + return + + if not self._check_auth(): + return + + length = int(self.headers.get("Content-Length", 0)) + raw_body = self.rfile.read(length) if length > 0 else b"" + form = parse_qs(raw_body.decode("utf-8", errors="replace")) + + # Only cookie-authenticated requests need a CSRF token. Basic Auth + # clients prove identity on every request so they're not subject to + # the CSRF attack vector. + if self._session_token() is not None: + if not self._check_csrf(form.get("csrf", [""])[0]): + self._send_bytes(HTTPStatus.FORBIDDEN, "text/plain; charset=utf-8", b"csrf token invalid") + return + + app = server.current_app() + containers = server.current_containers() + parts = path.strip("/").split("/") + + if len(parts) == 3 and parts[0] == "maintenance" and parts[2] in ("enter", "exit"): + name, action = parts[1], parts[2] + if name not in containers: + self._send_bytes(HTTPStatus.NOT_FOUND, "text/plain; charset=utf-8", b"unknown container") + return + if not app.maintenance.enabled: + self._send_bytes( + HTTPStatus.CONFLICT, + "text/plain; charset=utf-8", + b"maintenance windows are disabled in config", + ) + return + + marker_dir = Path(app.maintenance.dir) + marker = marker_dir / name + if action == "enter": + marker_dir.mkdir(parents=True, exist_ok=True) + marker.touch() + else: + marker.unlink(missing_ok=True) + + self.send_response(HTTPStatus.SEE_OTHER) + self.send_header("Location", "/") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if len(parts) == 3 and parts[0] == "notifications" and parts[2] in ("mute", "unmute"): + name, action = parts[1], parts[2] + if name not in containers: + self._send_bytes(HTTPStatus.NOT_FOUND, "text/plain; charset=utf-8", b"unknown container") + return + if not app.notification_mute.enabled: + self._send_bytes( + HTTPStatus.CONFLICT, + "text/plain; charset=utf-8", + b"notification muting is disabled in config", + ) + return + + marker_dir = Path(app.notification_mute.dir) + marker = marker_dir / name + if action == "mute": + marker_dir.mkdir(parents=True, exist_ok=True) + marker.touch() + else: + marker.unlink(missing_ok=True) + + self.send_response(HTTPStatus.SEE_OTHER) + self.send_header("Location", "/") + self.send_header("Content-Length", "0") + self.end_headers() + return + + if len(parts) == 2 and parts[0] == "settings": + section = parts[1] + try: + _apply_settings_update(server.config_ref, section, form) + except _SettingsError as exc: + flash = f"error:{exc}" + self.send_response(HTTPStatus.SEE_OTHER) + self.send_header("Location", f"/settings?flash={_url_quote(flash)}") + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(HTTPStatus.SEE_OTHER) + self.send_header("Location", f"/settings?flash={_url_quote('ok:' + section)}") + self.send_header("Content-Length", "0") + self.end_headers() + return + + self._send_bytes(HTTPStatus.NOT_FOUND, "text/plain; charset=utf-8", b"not found") + + +class DashboardServer: + """Optional embedded web dashboard. Always safe to construct and call: + a complete no-op when dashboard.enabled is false. Refuses to start + (logs an error) if enabled with an empty password, to avoid + accidentally exposing an unauthenticated dashboard. + """ + + def __init__( + self, + config_ref: ConfigRef, + states: dict[str, ContainerState], + maintenance: MaintenanceTracker, + notification_mute: NotificationMuteTracker, + ): + self._config_ref = config_ref + self._states = states + self._maintenance = maintenance + self._notification_mute = notification_mute + self._httpd: _DashboardHTTPServer | None = None + self._thread: threading.Thread | None = None + + @property + def enabled(self) -> bool: + return self._config_ref.get().dashboard.enabled + + def start(self) -> None: + dash_cfg = self._config_ref.get().dashboard + if not dash_cfg.enabled: + return + if not dash_cfg.password: + log.error( + "dashboard.enabled is true but dashboard.password is empty; " + "refusing to start the dashboard" + ) + return + + self._httpd = _DashboardHTTPServer( + (dash_cfg.host, dash_cfg.port), + _Handler, + config_ref=self._config_ref, + states=self._states, + maintenance=self._maintenance, + notification_mute=self._notification_mute, + ) + self._thread = threading.Thread( + target=self._httpd.serve_forever, name="dashboard", daemon=True + ) + self._thread.start() + log.info( + "Dashboard listening on %s:%d", + dash_cfg.host, + self._httpd.server_address[1], + ) + + def stop(self) -> None: + if self._httpd is None: + return + self._httpd.shutdown() + self._httpd.server_close() + if self._thread is not None: + self._thread.join(timeout=5) + self._httpd = None + self._thread = None diff --git a/src/inspector/docker_client.py b/src/inspector/docker_client.py new file mode 100755 index 0000000..3011147 --- /dev/null +++ b/src/inspector/docker_client.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import logging + +import docker +from docker.errors import APIError, NotFound + +from .health import CONTAINER_NOT_FOUND, ERROR, HealthResult, classify + + +log = logging.getLogger("inspector.docker") + + +def build_client() -> docker.DockerClient: + return docker.from_env() + + +def get_container_status(client: docker.DockerClient, name: str) -> HealthResult: + try: + container = client.containers.get(name) + return classify(name, container.attrs) + except NotFound: + return HealthResult( + container_name=name, + status=CONTAINER_NOT_FOUND, + message=f"container {name!r} not found", + ) + except APIError as exc: + log.error("Docker API error for %s: %s", name, exc) + return HealthResult( + container_name=name, + status=ERROR, + message=f"docker api error: {exc}", + ) + except Exception as exc: # defensive: never let the loop die + log.exception("Unexpected error inspecting %s: %s", name, exc) + return HealthResult( + container_name=name, + status=ERROR, + message=f"unexpected: {exc}", + ) + + +def restart_container(client: docker.DockerClient, name: str) -> tuple[bool, str | None]: + try: + container = client.containers.get(name) + container.restart() + return True, None + except NotFound: + return False, "container not found" + except APIError as exc: + return False, f"docker api error: {exc}" + except Exception as exc: + return False, f"unexpected: {exc}" + + +def start_container(client: docker.DockerClient, name: str) -> tuple[bool, str | None]: + try: + container = client.containers.get(name) + container.start() + return True, None + except NotFound: + return False, "container not found" + except APIError as exc: + return False, f"docker api error: {exc}" + except Exception as exc: + return False, f"unexpected: {exc}" diff --git a/src/inspector/eventstore.py b/src/inspector/eventstore.py new file mode 100755 index 0000000..216c83d --- /dev/null +++ b/src/inspector/eventstore.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import logging +import sqlite3 +from datetime import datetime, timedelta +from pathlib import Path + +from .config import ContainerConfig, DatabaseConfig +from .state import ContainerState, utcnow + + +log = logging.getLogger("inspector.eventstore") + + +EVENT_STATUS_CHANGE = "status_change" +EVENT_RECOVERY = "recovery" +EVENT_NOT_FOUND = "not_found" +EVENT_FAILURE = "failure" +EVENT_RESTART_LIMIT = "restart_limit" +EVENT_RESTART_DRYRUN = "restart_dry_run" +EVENT_RESTART = "restart" +EVENT_RESTART_FAILED = "restart_failed" +EVENT_PORTAINER_REDEPLOY = "portainer_redeploy" +EVENT_PORTAINER_REDEPLOY_FAILED = "portainer_redeploy_failed" +EVENT_PORTAINER_REDEPLOY_SKIPPED = "portainer_redeploy_skipped" +EVENT_MAINTENANCE_ENTER = "maintenance_enter" +EVENT_MAINTENANCE_EXIT = "maintenance_exit" +EVENT_NOTIFICATION_MUTE_ENTER = "notification_mute_enter" +EVENT_NOTIFICATION_MUTE_EXIT = "notification_mute_exit" + +# Event types considered "notable" for the health-history summary digest. +# EVENT_STATUS_CHANGE, EVENT_MAINTENANCE_ENTER, EVENT_MAINTENANCE_EXIT, +# EVENT_NOTIFICATION_MUTE_ENTER, and EVENT_NOTIFICATION_MUTE_EXIT are +# excluded: they're bookkeeping/transition markers, not failure activity, +# and would dominate the digest with noise. +SUMMARY_EVENT_TYPES = ( + EVENT_FAILURE, + EVENT_RESTART, + EVENT_RESTART_DRYRUN, + EVENT_RESTART_FAILED, + EVENT_RESTART_LIMIT, + EVENT_RECOVERY, + EVENT_NOT_FOUND, + EVENT_PORTAINER_REDEPLOY, + EVENT_PORTAINER_REDEPLOY_FAILED, + EVENT_PORTAINER_REDEPLOY_SKIPPED, +) + + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + container TEXT NOT NULL, + event_type TEXT NOT NULL, + old_status TEXT, + new_status TEXT, + error_count INTEGER, + message TEXT +); +CREATE INDEX IF NOT EXISTS idx_events_container_ts ON events (container, ts); +CREATE INDEX IF NOT EXISTS idx_events_ts ON events (ts); +""" + + +class EventStore: + """Durable per-container event history, backed by SQLite. + + Always safe to construct and call: when disabled, or if the database + file can't be opened, every method is a no-op. A database error is + logged but never crashes Inspector, same philosophy as GotifyNotifier. + """ + + def __init__(self, config: DatabaseConfig): + self._config = config + self._conn: sqlite3.Connection | None = None + self._last_prune: datetime | None = None + + if not config.enabled: + return + + try: + path = Path(config.path) + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(path)) + conn.execute("PRAGMA journal_mode=WAL") + conn.executescript(_SCHEMA) + self._conn = conn + log.info("Event history database opened: %s", path) + self.prune() + except Exception as exc: + log.error("Event history database unavailable (%s); continuing without it", exc) + self._conn = None + + @property + def enabled(self) -> bool: + return self._conn is not None + + def record( + self, + container: str, + event_type: str, + old_status: str | None = None, + new_status: str | None = None, + error_count: int | None = None, + message: str | None = None, + ) -> None: + if self._conn is None: + return + try: + self._conn.execute( + "INSERT INTO events " + "(ts, container, event_type, old_status, new_status, error_count, message) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + utcnow().isoformat(), + container, + event_type, + old_status, + new_status, + error_count, + message, + ), + ) + self._conn.commit() + except Exception as exc: + log.error("Failed to record event (%s/%s): %s", container, event_type, exc) + + def prune(self) -> None: + if self._conn is None: + return + try: + cutoff = (utcnow() - timedelta(days=self._config.retention_days)).isoformat() + cur = self._conn.execute("DELETE FROM events WHERE ts < ?", (cutoff,)) + self._conn.commit() + if cur.rowcount: + log.info( + "Event history: pruned %d row(s) older than %d days", + cur.rowcount, + self._config.retention_days, + ) + self._last_prune = utcnow() + except Exception as exc: + log.error("Event history prune failed: %s", exc) + + def maybe_prune(self) -> None: + if self._conn is None: + return + now = utcnow() + if self._last_prune is None or (now - self._last_prune) >= timedelta(hours=24): + self.prune() + + def summarize(self, since: datetime) -> dict[str, dict[str, int]]: + """Per-container counts of notable events at/after `since`. + + Returns {container: {event_type: count}}. Containers with zero + matching events are absent from the result. Returns {} if disabled + or on any error. + """ + if self._conn is None: + return {} + try: + placeholders = ",".join("?" for _ in SUMMARY_EVENT_TYPES) + cur = self._conn.execute( + f"SELECT container, event_type, COUNT(*) FROM events " + f"WHERE ts >= ? AND event_type IN ({placeholders}) " + f"GROUP BY container, event_type", + (since.isoformat(), *SUMMARY_EVENT_TYPES), + ) + result: dict[str, dict[str, int]] = {} + for container, event_type, count in cur.fetchall(): + result.setdefault(container, {})[event_type] = count + return result + except Exception as exc: + log.error("Event history summarize failed: %s", exc) + return {} + + def restore_last_restarts( + self, + states: dict[str, ContainerState], + configs: dict[str, ContainerConfig], + ) -> None: + """Backfill last_restart_at (and any still-active cooldown) from the + most recent recorded restart per container. + + Called once on startup so the dashboard doesn't show "Never" just + because Inspector itself restarted, even though the container's + last actual restart is recorded in the event history. No-op when + disabled or on any error. + """ + if self._conn is None: + return + now = utcnow() + for name, state in states.items(): + try: + cur = self._conn.execute( + "SELECT ts FROM events WHERE container = ? AND event_type = ? " + "ORDER BY ts DESC LIMIT 1", + (name, EVENT_RESTART), + ) + row = cur.fetchone() + except Exception as exc: + log.error("Event history: failed to read last restart for %s: %s", name, exc) + continue + if row is None: + continue + last_restart = datetime.fromisoformat(row[0]) + state.last_restart_at = last_restart + state.restart_history.append(last_restart) + cooldown_until = last_restart + timedelta(seconds=configs[name].cooldown_seconds) + if cooldown_until > now: + state.cooldown_until = cooldown_until + + def close(self) -> None: + if self._conn is not None: + try: + self._conn.close() + finally: + self._conn = None diff --git a/src/inspector/formatting.py b/src/inspector/formatting.py new file mode 100755 index 0000000..c4655ae --- /dev/null +++ b/src/inspector/formatting.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from zoneinfo import ZoneInfo + + +PACIFIC = ZoneInfo("America/Los_Angeles") + + +def format_timestamp(value: datetime | str | None) -> str: + """Render a timestamp in US Pacific time, e.g. '06-14 03:45 PM'. + + Accepts a datetime, an ISO 8601 string (as stored by EventStore / + ContainerState), or None/"" (returns ""). Naive datetimes/strings are + assumed to be UTC (matching utcnow()/EventStore storage). Unparseable + strings are returned unchanged. + """ + if not value: + return "" + if isinstance(value, str): + try: + value = datetime.fromisoformat(value) + except ValueError: + return value + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(PACIFIC).strftime("%m-%d %I:%M %p") diff --git a/src/inspector/health.py b/src/inspector/health.py new file mode 100755 index 0000000..7a0c0ec --- /dev/null +++ b/src/inspector/health.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +HEALTHY = "HEALTHY" +UNHEALTHY = "UNHEALTHY" +STARTING = "STARTING" +NO_HEALTH_CHECK = "NO HEALTH CHECK" +NOT_RUNNING = "NOT RUNNING" +CONTAINER_NOT_FOUND = "CONTAINER NOT FOUND" +ERROR = "ERROR" +UNKNOWN = "UNKNOWN" +UNREACHABLE = "UNREACHABLE" + + +@dataclass(frozen=True) +class HealthResult: + container_name: str + status: str + docker_status: str | None = None + health_status: str | None = None + message: str | None = None + + +def classify(container_name: str, inspect_data: dict[str, Any]) -> HealthResult: + state = (inspect_data or {}).get("State") or {} + docker_status = state.get("Status") + health = (state.get("Health") or {}).get("Status") + + if docker_status is None: + return HealthResult( + container_name=container_name, + status=UNKNOWN, + docker_status=docker_status, + health_status=health, + message="missing State.Status", + ) + + if docker_status != "running": + return HealthResult( + container_name=container_name, + status=NOT_RUNNING, + docker_status=docker_status, + health_status=health, + message=f"docker status: {docker_status}", + ) + + if health is None: + return HealthResult( + container_name=container_name, + status=NO_HEALTH_CHECK, + docker_status=docker_status, + health_status=None, + ) + + if health == "healthy": + return HealthResult(container_name, HEALTHY, docker_status, health) + if health == "unhealthy": + return HealthResult(container_name, UNHEALTHY, docker_status, health) + if health == "starting": + return HealthResult(container_name, STARTING, docker_status, health) + + return HealthResult( + container_name=container_name, + status=UNKNOWN, + docker_status=docker_status, + health_status=health, + message=f"unknown health status: {health}", + ) diff --git a/src/inspector/health_summary.py b/src/inspector/health_summary.py new file mode 100755 index 0000000..4fc59eb --- /dev/null +++ b/src/inspector/health_summary.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import logging +from datetime import datetime, timedelta + +from .config import HealthSummaryConfig +from .eventstore import SUMMARY_EVENT_TYPES, EventStore +from .notifier import GotifyNotifier +from .state import utcnow + + +log = logging.getLogger("inspector.health_summary") + + +_LABELS = { + "failure": "failures", + "restart": "restarts", + "restart_dry_run": "dry-run restarts", + "restart_failed": "restart failures", + "restart_limit": "restart-limit hits", + "recovery": "recoveries", + "not_found": "not-found events", + "portainer_redeploy": "Portainer redeploys", + "portainer_redeploy_failed": "Portainer redeploy failures", + "portainer_redeploy_skipped": "Portainer redeploys skipped", +} + + +class HealthSummaryReporter: + """Periodic digest of per-container health activity from EventStore. + + Always safe to construct and call: no-ops when health_summary.enabled + is false. If enabled but EventStore isn't recording (database.enabled + is false, or the database failed to open), logs a one-time startup + warning and digests will always show "no issues". + """ + + def __init__(self, config: HealthSummaryConfig, eventstore: EventStore, notifier: GotifyNotifier): + self._config = config + self._eventstore = eventstore + self._notifier = notifier + self._last_report: datetime | None = None + + if config.enabled and not eventstore.enabled: + log.warning( + "health_summary.enabled is true but database.enabled is false " + "(or the event database failed to open); digests will report " + "no data until event history is enabled" + ) + + @property + def enabled(self) -> bool: + return self._config.enabled + + def maybe_report(self, container_names: list[str]) -> None: + if not self.enabled: + return + now = utcnow() + if self._last_report is not None and (now - self._last_report) < timedelta(hours=self._config.interval_hours): + return + + since = self._last_report or (now - timedelta(hours=self._config.interval_hours)) + self._report(container_names, since=since, now=now) + self._last_report = now + + def _report(self, container_names: list[str], since: datetime, now: datetime) -> None: + counts = self._eventstore.summarize(since) + + lines: list[str] = [] + md_lines: list[str] = [] + any_activity = False + for name in sorted(container_names): + c = counts.get(name, {}) + parts = [f"{_LABELS[t]}={c[t]}" for t in SUMMARY_EVENT_TYPES if c.get(t)] + if parts: + any_activity = True + detail = ", ".join(parts) + else: + detail = "no issues" + lines.append(f"{name}: {detail}") + md_lines.append(f"- **{name}**: {detail}") + + hours = max(1, round((now - since).total_seconds() / 3600)) + log.info("Health summary (last %dh):", hours) + for line in lines: + log.info(" %s", line) + + if any_activity: + body = f"### 📋 Health summary — last {hours}h\n\n" + "\n".join(md_lines) + self._notifier.notify( + title="📋 Inspector: health summary", + message=body, + priority=self._config.priority, + ) diff --git a/src/inspector/http_check.py b/src/inspector/http_check.py new file mode 100755 index 0000000..6dcc441 --- /dev/null +++ b/src/inspector/http_check.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass + +import requests + +from .config import HttpCheckConfig + +log = logging.getLogger("inspector.http_check") + + +@dataclass(frozen=True) +class HttpCheckResult: + ok: bool + status_code: int = 0 # synthetic 0 == connection-level failure (refused/timeout/DNS) + message: str | None = None + + +def check_http(config: HttpCheckConfig) -> HttpCheckResult: + """GET config.url and classify the outcome into three tiers: + + - status_code == expected_status -> ok=True (up) + - status_code == 0 -> ok=False (unreachable: connection + refused / timeout / DNS failure) + - any other code -> ok=False (responding, but the + status code is unexpected) + + Never raises. Connection-level failures return before any comparison to + expected_status, so they can never be mistaken for "up". + """ + url = config.url + try: + resp = requests.get(url, timeout=config.timeout_seconds) + except requests.Timeout: + return HttpCheckResult( + ok=False, status_code=0, message=f"HTTP check unreachable (timed out): {url}" + ) + except requests.ConnectionError as exc: + return HttpCheckResult( + ok=False, + status_code=0, + message=f"HTTP check unreachable (connection failed): {url} ({exc})", + ) + except requests.RequestException as exc: + return HttpCheckResult( + ok=False, status_code=0, message=f"HTTP check failed: {url} ({exc})" + ) + except Exception as exc: + log.exception("Unexpected error in http_check for %s: %s", url, exc) + return HttpCheckResult( + ok=False, status_code=0, message=f"HTTP check error: {url} ({exc})" + ) + + code = resp.status_code + if code == config.expected_status: + return HttpCheckResult(ok=True, status_code=code) + return HttpCheckResult( + ok=False, + status_code=code, + message=( + f"HTTP check unexpected status: {url} returned {code}, " + f"expected {config.expected_status} (responding but unexpected)" + ), + ) diff --git a/src/inspector/logging_setup.py b/src/inspector/logging_setup.py new file mode 100755 index 0000000..2d09f0a --- /dev/null +++ b/src/inspector/logging_setup.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import logging +import os +from logging.handlers import TimedRotatingFileHandler +from pathlib import Path + +import colorlog + +from .config import InspectorConfig + + +class _BlankLineFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + if record.getMessage() == "": + return "" + return super().format(record) + + +def setup_logging(config: InspectorConfig) -> logging.Logger: + class _BlankLineColoredFormatter(colorlog.ColoredFormatter): + def format(self, record: logging.LogRecord) -> str: + if record.getMessage() == "": + return "" + return super().format(record) + + log_dir = Path(config.log_dir) + log_dir.mkdir(parents=True, exist_ok=True) + log_path = log_dir / config.log_filename + + root = logging.getLogger() + root.setLevel(config.log_level) + + for handler in list(root.handlers): + root.removeHandler(handler) + + file_handler = TimedRotatingFileHandler( + filename=str(log_path), + when="midnight", + interval=1, + backupCount=config.log_retention_days, + encoding="utf-8", + utc=False, + ) + file_handler.setFormatter( + _BlankLineFormatter( + fmt="%(asctime)s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + root.addHandler(file_handler) + + console_handler = colorlog.StreamHandler() + console_handler.setFormatter( + _BlankLineColoredFormatter( + fmt="%(log_color)s%(asctime)s %(levelname)-8s%(reset)s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + log_colors={ + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "bold_red", + }, + ) + ) + root.addHandler(console_handler) + + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("docker").setLevel(logging.WARNING) + + logger = logging.getLogger("inspector") + logger.info("Logging initialized: file=%s level=%s", log_path, config.log_level) + return logger diff --git a/src/inspector/main.py b/src/inspector/main.py new file mode 100755 index 0000000..c0c55a8 --- /dev/null +++ b/src/inspector/main.py @@ -0,0 +1,519 @@ +from __future__ import annotations + +import argparse +import logging +import signal +import sys +import time +from datetime import datetime +from pathlib import Path + +from . import __version__ +from .config import AppConfig, ConfigRef, ContainerConfig, load_config, load_raw +from .dashboard import DashboardServer +from .docker_client import build_client, get_container_status +from .eventstore import ( + EVENT_FAILURE, + EVENT_NOT_FOUND, + EVENT_RECOVERY, + EVENT_RESTART_LIMIT, + EVENT_STATUS_CHANGE, + EventStore, +) +from .formatting import format_timestamp +from .health import ( + CONTAINER_NOT_FOUND, + HEALTHY, + NO_HEALTH_CHECK, + NOT_RUNNING, + STARTING, + UNHEALTHY, + UNKNOWN, + UNREACHABLE, + HealthResult, +) +from .health_summary import HealthSummaryReporter +from .http_check import check_http +from .logging_setup import setup_logging +from .maintenance import MaintenanceTracker +from .notification_mute import NotificationMuteTracker +from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier +from .portainer_client import PortainerClient +from .recovery import perform_recovery +from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient +from .state import ( + ContainerState, + should_count_failure, + should_restart, + utcnow, +) + + +log = logging.getLogger("inspector") + + +_stop = False + + +def _request_stop(signum, _frame): + global _stop + log.info("Received signal %s; shutting down after current loop", signum) + _stop = True + + +def _apply_reachability_override( + container_config: ContainerConfig, + result: HealthResult, + uptime_kuma_snapshot: HeartbeatSnapshot | None, +) -> HealthResult: + """Override an otherwise-OK Docker status with UNREACHABLE if an + enabled per-container reachability check (HTTP or Uptime Kuma) says the + container can't actually be reached. + + Docker failure statuses (UNHEALTHY, NOT_RUNNING, ERROR, + CONTAINER_NOT_FOUND, UNKNOWN) are never masked. HTTP check takes + precedence over Uptime Kuma if both are enabled and failing. + """ + if result.status not in (HEALTHY, STARTING, NO_HEALTH_CHECK): + return result + + name = container_config.name + + http_cfg = container_config.http_check + if http_cfg.enabled: + check = check_http(http_cfg) + if not check.ok: + return HealthResult( + container_name=name, + status=UNREACHABLE, + docker_status=result.docker_status, + health_status=result.health_status, + message=check.message, + ) + + uk_cfg = container_config.uptime_kuma + if uk_cfg.enabled and uptime_kuma_snapshot is not None: + if uptime_kuma_snapshot.is_up(uk_cfg.monitor_name) is False: + return HealthResult( + container_name=name, + status=UNREACHABLE, + docker_status=result.docker_status, + health_status=result.health_status, + message=f"Uptime Kuma monitor '{uk_cfg.monitor_name}' is down", + ) + + return result + + +def _check_one( + container_config: ContainerConfig, + state: ContainerState, + result: HealthResult, + notifier: GotifyNotifier, + eventstore: EventStore, + maintenance: MaintenanceTracker, + notification_mute: NotificationMuteTracker, + client, + dry_run: bool, + width: int, + uptime_kuma_snapshot: HeartbeatSnapshot | None, + app: AppConfig, + states: dict[str, ContainerState], + portainer_client: PortainerClient, + stack_cooldowns: dict[str, datetime], +) -> None: + name = container_config.name + now = utcnow() + state.last_check_at = now + + state.previous_status = state.current_status + state.current_status = result.status + + result = _apply_reachability_override(container_config, result, uptime_kuma_snapshot) + state.current_status = result.status # may now be UNREACHABLE + + if maintenance.is_active(name): + return + + if notification_mute.is_muted(name): + notifier = NullNotifier() + + if state.previous_status != state.current_status: + log.info( + "%-*s status: %s -> %s%s", + width, + name, + state.previous_status or "INIT", + state.current_status, + f" ({result.message})" if result.message else "", + ) + eventstore.record( + container=name, + event_type=EVENT_STATUS_CHANGE, + old_status=state.previous_status, + new_status=state.current_status, + error_count=state.error_count, + message=result.message, + ) + + if ( + state.previous_status is not None + and state.previous_status != HEALTHY + and state.current_status == HEALTHY + and state.last_notification_state != HEALTHY + ): + log.info("%-*s recovered: %s -> HEALTHY", width, name, state.previous_status) + notifier.notify( + title=f"✅ Inspector: {name} recovered", + message=( + f"**{name}** is back to **HEALTHY** (was *{state.previous_status}*).\n\n" + f"- Errors before recovery: **{state.error_count}**\n" + f"- Time: {format_timestamp(now)}" + ), + priority=3, + ) + eventstore.record( + container=name, + event_type=EVENT_RECOVERY, + old_status=state.previous_status, + new_status=HEALTHY, + error_count=state.error_count, + ) + state.last_notification_state = HEALTHY + state.error_count = 0 + state.restart_limit_notified = False + state.restarted_this_episode = False + log.info("") + + if result.status == CONTAINER_NOT_FOUND: + if state.last_notification_state != CONTAINER_NOT_FOUND: + log.error("%-*s container not found", width, name) + notifier.notify( + title=f"❓ Inspector: {name} not found", + message=( + f"**{name}** is missing from Docker.\n\n" + + (f"- Detail: {result.message}\n" if result.message else "") + + f"- Last known status: *{state.previous_status or 'unknown'}*\n" + f"- Time: {format_timestamp(now)}" + ), + priority=7, + ) + eventstore.record( + container=name, + event_type=EVENT_NOT_FOUND, + old_status=state.previous_status, + new_status=CONTAINER_NOT_FOUND, + message=result.message, + ) + state.last_notification_state = CONTAINER_NOT_FOUND + return + + in_grace = state.in_grace(now, container_config.grace_period_seconds) + if in_grace and result.status != HEALTHY: + log.debug("%-*s in grace period; not counting %s", width, name, result.status) + + if state.in_cooldown(now): + remaining = int((state.cooldown_until - now).total_seconds()) + log.debug("%-*s in cooldown for %ss", width, name, remaining) + return + + if should_count_failure(result.status, container_config, in_grace): + state.error_count += 1 + if state.error_count == 1: + log.info("") + log.warning( + "%-*s failure: status=%s count=%d/%d", + width, + name, + result.status, + state.error_count, + container_config.max_error_count, + ) + if state.error_count == 1 or state.last_notification_state != result.status: + notifier.notify( + title=f"🚨 Inspector: {name} is {result.status}", + message=( + f"**{name}** is **{result.status}**" + + (f" — {result.message}" if result.message else "") + + ".\n\n" + f"- Error count: **{state.error_count}/{container_config.max_error_count}**\n" + f"- Previous status: *{state.previous_status or 'unknown'}*\n" + f"- Time: {format_timestamp(now)}" + ), + priority=5, + ) + eventstore.record( + container=name, + event_type=EVENT_FAILURE, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=result.message, + ) + state.last_notification_state = result.status + elif result.status == HEALTHY: + if state.error_count != 0: + log.info("%-*s healthy again; resetting error count", width, name) + state.error_count = 0 + state.restarted_this_episode = False + + if state.error_count >= container_config.max_error_count: + ok, reason = should_restart(state, container_config, now) + if not ok and reason == "hourly restart limit reached": + if not state.restart_limit_notified: + log.critical( + "%-*s exceeded max_restarts_per_hour (%d); pausing restarts", + width, + name, + container_config.max_restarts_per_hour, + ) + notifier.notify( + title=f"🛑 Inspector: {name} restart limit reached", + message=( + f"**{name}** hit its restart limit: " + f"**{container_config.max_restarts_per_hour}** restart(s)/hour.\n\n" + f"- Current status: **{result.status}**\n" + f"- Restarts paused until the hourly window clears\n" + f"- Time: {format_timestamp(now)}" + ), + priority=9, + ) + eventstore.record( + container=name, + event_type=EVENT_RESTART_LIMIT, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"max_restarts_per_hour={container_config.max_restarts_per_hour}", + ) + state.restart_limit_notified = True + return + + if ok: + dep = container_config.depends_on + if dep: + dep_status = states[dep].current_status if dep in states else None + if dep_status not in (HEALTHY, STARTING, NO_HEALTH_CHECK): + if container_config.vpn_routed: + log.info( + "%-*s is vpn_routed and depends on %s; deferring recovery " + "because %s is %s", + width, name, dep, dep, dep_status, + ) + return + + perform_recovery( + app=app, + container_config=container_config, + state=state, + result=result, + notifier=notifier, + eventstore=eventstore, + portainer_client=portainer_client, + stack_cooldowns=stack_cooldowns, + client=client, + dry_run=dry_run, + width=width, + now=now, + ) + + +def _topological_order(enabled_configs: dict[str, ContainerConfig]) -> list[str]: + """Order enabled container names so a container with `depends_on` + referencing another *enabled* container is checked after its dependency, + preserving relative config-file order otherwise. + + Falls back to config order (with a warning) on a dependency cycle, or + per-container (with a warning) if `depends_on` points to a disabled or + not-enabled container. + """ + names = list(enabled_configs.keys()) + deps: dict[str, str | None] = {} + for name in names: + dep = enabled_configs[name].depends_on + if dep and dep in enabled_configs: + deps[name] = dep + else: + if dep and dep not in enabled_configs: + log.warning( + "%s depends_on %r, which is disabled or not found among enabled " + "containers; ignoring dependency ordering for %s", + name, dep, name, + ) + deps[name] = None + + order: list[str] = [] + visiting: set[str] = set() + visited: set[str] = set() + cycle_detected = False + + def visit(n: str) -> None: + nonlocal cycle_detected + if n in visited: + return + if n in visiting: + cycle_detected = True + return + visiting.add(n) + dep = deps.get(n) + if dep is not None: + visit(dep) + visiting.discard(n) + visited.add(n) + order.append(n) + + for name in names: + visit(name) + if cycle_detected: + log.warning( + "Dependency cycle detected in container 'depends_on' configuration; " + "falling back to config-file order" + ) + return names + + return order + + +def _build_notifier(app: AppConfig) -> MultiNotifier: + return MultiNotifier([GotifyNotifier(app.gotify), PushoverNotifier(app.pushover)]) + + +def run_loop(config_ref: ConfigRef) -> int: + app = config_ref.get() + notifier = _build_notifier(app) + uptime_kuma_client = UptimeKumaClient(app.integrations.uptime_kuma) + eventstore = EventStore(app.database) + health_summary = HealthSummaryReporter(app.health_summary, eventstore, notifier) + maintenance = MaintenanceTracker( + app.maintenance, + eventstore, + scheduled_containers={c.name for c in app.containers if c.maintenance_window}, + ) + notification_mute = NotificationMuteTracker(app.notification_mute, eventstore) + + try: + client = build_client() + client.ping() + log.info("Docker connection successful") + except Exception as exc: + if "Permission denied" in str(exc): + log.critical( + "Permission denied on /var/run/docker.sock. Set DOCKER_GID to " + "the host docker group GID: stat -c '%g' /var/run/docker.sock" + ) + else: + log.critical("Cannot reach Docker daemon: %s", exc) + return 2 + + states: dict[str, ContainerState] = { + c.name: ContainerState(name=c.name) for c in app.containers if c.enabled + } + enabled_configs = {c.name: c for c in app.containers if c.enabled} + name_width = max((len(name) for name in enabled_configs), default=0) + eventstore.restore_last_restarts(states, enabled_configs) + + check_order = _topological_order(enabled_configs) + portainer_client = PortainerClient(app.integrations.portainer) + stack_cooldowns: dict[str, datetime] = {} + + dashboard = DashboardServer( + config_ref, + states, + maintenance, + notification_mute, + ) + dashboard.start() + + log.info( + "Inspector %s started; watching %d container(s); interval=%ss dry_run=%s", + __version__, + len(states), + app.inspector.interval_seconds, + app.inspector.dry_run, + ) + + while not _stop: + app = config_ref.get() # re-read each iteration so settings-page edits take effect + notifier = _build_notifier(app) + + # Rebuild enabled_configs / check_order from current config so edits + # to per-container settings apply next cycle. Containers added via + # the settings page are logged as 'requires restart' because the + # states dict is built once at startup. + current_enabled = {c.name: c for c in app.containers if c.enabled} + new_names = current_enabled.keys() - states.keys() + if new_names: + log.warning( + "Containers added in config require Inspector restart to begin monitoring: %s", + sorted(new_names), + ) + enabled_configs = {n: c for n, c in current_enabled.items() if n in states} + check_order = _topological_order(enabled_configs) + + uptime_kuma_snapshot = uptime_kuma_client.fetch() + for name in check_order: + cfg = enabled_configs[name] + result = get_container_status(client, name) + try: + _check_one( + container_config=cfg, + state=states[name], + result=result, + notifier=notifier, + eventstore=eventstore, + maintenance=maintenance, + notification_mute=notification_mute, + client=client, + dry_run=app.inspector.dry_run, + width=name_width, + uptime_kuma_snapshot=uptime_kuma_snapshot, + app=app, + states=states, + portainer_client=portainer_client, + stack_cooldowns=stack_cooldowns, + ) + except Exception as exc: + log.exception("%-*s error processing check: %s", name_width, name, exc) + + if app.inspector.heartbeat: + log.info("heartbeat: %d containers checked", len(states)) + + eventstore.maybe_prune() + health_summary.maybe_report(list(enabled_configs.keys())) + + # interruptible sleep + slept = 0 + while slept < app.inspector.interval_seconds and not _stop: + time.sleep(min(1, app.inspector.interval_seconds - slept)) + slept += 1 + + dashboard.stop() + eventstore.close() + log.info("Inspector stopped") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="inspector", description="Docker container health watcher") + parser.add_argument("--config", "-c", default="/app/config.yml", help="Path to config.yml") + parser.add_argument("--version", action="version", version=f"inspector {__version__}") + args = parser.parse_args(argv) + + config_path = Path(args.config) + if not config_path.exists(): + print(f"config file not found: {config_path}", file=sys.stderr) + return 2 + + app = load_config(config_path) + raw = load_raw(config_path) + config_ref = ConfigRef(config_path, app, raw) + setup_logging(app.inspector) + + signal.signal(signal.SIGTERM, _request_stop) + signal.signal(signal.SIGINT, _request_stop) + + return run_loop(config_ref) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inspector/maintenance.py b/src/inspector/maintenance.py new file mode 100755 index 0000000..cfa8091 --- /dev/null +++ b/src/inspector/maintenance.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import logging +from datetime import datetime, time, timedelta +from pathlib import Path + +from .config import MaintenanceConfig +from .eventstore import EVENT_MAINTENANCE_ENTER, EVENT_MAINTENANCE_EXIT, EventStore +from .state import localnow, utcnow + + +log = logging.getLogger("inspector.maintenance") + + +def _parse_hhmm(value: str) -> time: + hour, minute = value.split(":") + return time(int(hour), int(minute)) + + +def _in_window(now_time: time, start: time, end: time) -> bool: + if start <= end: + return start <= now_time < end + return now_time >= start or now_time < end # wraps past midnight + + +def _next_window(now: datetime, start: time, end: time) -> tuple[datetime, datetime]: + """Return the recurring daily maintenance window as (start, end) datetimes, + in the same (naive) timezone as ``now``. If ``now`` is inside a window, that + active window is returned; otherwise the next upcoming one. Handles both + same-day and past-midnight (start > end) windows. + + Caller must ensure start != end (a zero/full-day window is meaningless here). + """ + start_secs = start.hour * 3600 + start.minute * 60 + start.second + end_secs = end.hour * 3600 + end.minute * 60 + end.second + duration = timedelta(seconds=(end_secs - start_secs) % 86400) + + today = now.date() + candidates = [ + datetime.combine(today + timedelta(days=offset), start) + for offset in (-1, 0, 1) + ] + upcoming: list[datetime] = [] + for s in candidates: + if s <= now < s + duration: + return s, s + duration + if s > now: + upcoming.append(s) + s = min(upcoming) + return s, s + duration + + +class MaintenanceTracker: + """Per-container maintenance windows: recurring daily schedule OR + ad-hoc marker files. Always safe to construct and call: a complete + no-op when maintenance.enabled is false. + + - Recurring window: containers with maintenance_window: true in their + config are "in maintenance" every day between maintenance.start and + maintenance.end (local time, HH:MM, may wrap past midnight). + - Marker files: touch / to start, rm to + end. A marker older than maintenance.max_hours auto-expires. + + Logs one INFO line on enter/exit and records + EVENT_MAINTENANCE_ENTER/EVENT_MAINTENANCE_EXIT (if event history is + enabled) on each transition. + """ + + def __init__(self, config: MaintenanceConfig, eventstore: EventStore, scheduled_containers: set[str]): + self._config = config + self._eventstore = eventstore + self._scheduled = scheduled_containers + self._start = _parse_hhmm(config.start) + self._end = _parse_hhmm(config.end) + self._active: dict[str, bool] = {} + + @property + def enabled(self) -> bool: + return self._config.enabled + + def is_active(self, name: str) -> bool: + if not self.enabled: + return False + + active = self._marker_active(name) or self._scheduled_active(name) + + was_active = self._active.get(name, False) + if active != was_active: + self._log_transition(name, active) + self._active[name] = active + return active + + def peek(self, name: str) -> bool: + """Read-only maintenance check for display purposes (e.g. the + dashboard). No transition logging, no eventstore writes, and no + update to internal tracking state. Safe to call from another + thread.""" + if not self.enabled: + return False + return self._marker_active(name) or self._scheduled_active(name) + + def next_scheduled_window(self, name: str) -> tuple[datetime, datetime] | None: + """Return the current or next recurring maintenance window for a + scheduled container as (start, end) naive-local datetimes, or None if + maintenance is disabled, the container has no recurring window, or the + window is empty (start == end). Read-only; safe to call from the + dashboard thread.""" + if not self.enabled or name not in self._scheduled or self._start == self._end: + return None + return _next_window(localnow(), self._start, self._end) + + def _scheduled_active(self, name: str) -> bool: + if name not in self._scheduled: + return False + return _in_window(localnow().time(), self._start, self._end) + + def _marker_active(self, name: str) -> bool: + marker = Path(self._config.dir) / name + try: + if not marker.is_file(): + return False + age_hours = (utcnow().timestamp() - marker.stat().st_mtime) / 3600 + if age_hours > self._config.max_hours: + if self._active.get(name, False): + log.warning( + "%s maintenance marker expired after %.1fh (max_hours=%d); " + "resuming normal monitoring", + name, age_hours, self._config.max_hours, + ) + return False + return True + except Exception as exc: + log.error("%s maintenance marker check failed: %s", name, exc) + return False + + def _log_transition(self, name: str, active: bool) -> None: + if active: + log.info("%s entering maintenance window", name) + self._eventstore.record(container=name, event_type=EVENT_MAINTENANCE_ENTER) + else: + log.info("%s exiting maintenance window; resuming normal monitoring", name) + self._eventstore.record(container=name, event_type=EVENT_MAINTENANCE_EXIT) diff --git a/src/inspector/notification_mute.py b/src/inspector/notification_mute.py new file mode 100755 index 0000000..10ce5cb --- /dev/null +++ b/src/inspector/notification_mute.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from .config import NotificationMuteConfig +from .eventstore import EVENT_NOTIFICATION_MUTE_ENTER, EVENT_NOTIFICATION_MUTE_EXIT, EventStore +from .state import utcnow + + +log = logging.getLogger("inspector.notification_mute") + + +class NotificationMuteTracker: + """Per-container ad-hoc notification mute via marker files under + notification_mute.dir. Always safe to construct and call: a complete + no-op when notification_mute.enabled is false. + + Muting silences outbound Gotify notifications only -- monitoring, + failure counting, cooldowns, restarts/recovery, and event-history + recording continue unchanged. A marker older than + notification_mute.default_hours auto-expires. + + Logs one INFO line on mute/unmute and records + EVENT_NOTIFICATION_MUTE_ENTER/EVENT_NOTIFICATION_MUTE_EXIT (if event + history is enabled) on each transition. + """ + + def __init__(self, config: NotificationMuteConfig, eventstore: EventStore): + self._config = config + self._eventstore = eventstore + self._active: dict[str, bool] = {} + + @property + def enabled(self) -> bool: + return self._config.enabled + + def is_muted(self, name: str) -> bool: + if not self.enabled: + return False + + active = self._marker_expiry(name) is not None + + was_active = self._active.get(name, False) + if active != was_active: + self._log_transition(name, active) + self._active[name] = active + return active + + def peek(self, name: str) -> bool: + """Read-only mute check for display (e.g. the dashboard). No + transition logging, no eventstore writes, no state update. Safe to + call from another thread.""" + if not self.enabled: + return False + return self._marker_expiry(name) is not None + + def muted_until(self, name: str) -> datetime | None: + """Expiry time for an active mute, or None if not muted/disabled.""" + if not self.enabled: + return None + return self._marker_expiry(name) + + def _marker_expiry(self, name: str) -> datetime | None: + marker = Path(self._config.dir) / name + try: + if not marker.is_file(): + return None + mtime = marker.stat().st_mtime + age_hours = (utcnow().timestamp() - mtime) / 3600 + if age_hours > self._config.default_hours: + if self._active.get(name, False): + log.warning( + "%s notification mute expired after %.1fh (default_hours=%d); " + "alerts resumed", + name, age_hours, self._config.default_hours, + ) + return None + return datetime.fromtimestamp(mtime, tz=timezone.utc) + timedelta( + hours=self._config.default_hours + ) + except Exception as exc: + log.error("%s notification mute marker check failed: %s", name, exc) + return None + + def _log_transition(self, name: str, active: bool) -> None: + if active: + log.info("%s notifications muted", name) + self._eventstore.record(container=name, event_type=EVENT_NOTIFICATION_MUTE_ENTER) + else: + log.info("%s notifications unmuted; alerts resumed", name) + self._eventstore.record(container=name, event_type=EVENT_NOTIFICATION_MUTE_EXIT) diff --git a/src/inspector/notifier.py b/src/inspector/notifier.py new file mode 100755 index 0000000..5538c1b --- /dev/null +++ b/src/inspector/notifier.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import logging +from typing import Iterable + +import requests + +from .config import GotifyConfig, PushoverConfig + + +log = logging.getLogger("inspector.notify") + + +class GotifyNotifier: + def __init__(self, config: GotifyConfig, timeout: float = 5.0): + self._config = config + self._timeout = timeout + + @property + def enabled(self) -> bool: + return self._config.enabled and bool(self._config.url) and bool(self._config.token) + + def notify(self, title: str, message: str, priority: int | None = None) -> None: + if not self.enabled: + return + url = f"{self._config.url}/message" + payload = { + "title": title, + "message": message, + "priority": self._config.priority if priority is None else priority, + "extras": { + "client::display": {"contentType": "text/markdown"}, + }, + } + try: + resp = requests.post( + url, + params={"token": self._config.token}, + json=payload, + timeout=self._timeout, + ) + if resp.status_code >= 400: + log.error( + "Gotify notification failed: status=%s body=%s", + resp.status_code, + resp.text[:200], + ) + except requests.RequestException as exc: + log.error("Gotify notification failed: %s", exc) + except Exception as exc: + log.exception("Gotify notifier crashed: %s", exc) + + +# Pushover's priority range is -2..2. Inspector uses a Gotify-style scale +# (roughly 1..10). Mapping chosen 2026-10-26 (see plan file). +# priority 2 ("emergency") requires retry+expire; defaults below. +_PUSHOVER_EMERGENCY_RETRY_SECONDS = 60 +_PUSHOVER_EMERGENCY_EXPIRE_SECONDS = 3600 + + +def _map_priority_to_pushover(inspector_priority: int) -> int: + if inspector_priority <= 2: + return -1 + if inspector_priority <= 4: + return 0 + if inspector_priority <= 6: + return 1 + return 2 + + +class PushoverNotifier: + API_URL = "https://api.pushover.net/1/messages.json" + + def __init__(self, config: PushoverConfig, timeout: float = 5.0): + self._config = config + self._timeout = timeout + + @property + def enabled(self) -> bool: + return ( + self._config.enabled + and bool(self._config.token) + and bool(self._config.user_key) + ) + + def notify(self, title: str, message: str, priority: int | None = None) -> None: + if not self.enabled: + return + + inspector_priority = self._config.priority if priority is None else priority + pushover_priority = _map_priority_to_pushover(inspector_priority) + + data = { + "token": self._config.token, + "user": self._config.user_key, + "title": title, + "message": message, + "priority": pushover_priority, + } + if self._config.device: + data["device"] = self._config.device + if self._config.sound: + data["sound"] = self._config.sound + if pushover_priority == 2: + data["retry"] = _PUSHOVER_EMERGENCY_RETRY_SECONDS + data["expire"] = _PUSHOVER_EMERGENCY_EXPIRE_SECONDS + + try: + resp = requests.post(self.API_URL, data=data, timeout=self._timeout) + if resp.status_code >= 400: + log.error( + "Pushover notification failed: status=%s body=%s", + resp.status_code, + resp.text[:200], + ) + except requests.RequestException as exc: + log.error("Pushover notification failed: %s", exc) + except Exception as exc: + log.exception("Pushover notifier crashed: %s", exc) + + +class MultiNotifier: + """Fan out notify() to every enabled child notifier. + + Enabled is true if any child is enabled. A failure in one child never + prevents the others from running. + """ + + def __init__(self, notifiers: Iterable[object]): + self._notifiers = list(notifiers) + + @property + def enabled(self) -> bool: + return any(getattr(n, "enabled", False) for n in self._notifiers) + + def notify(self, title: str, message: str, priority: int | None = None) -> None: + for n in self._notifiers: + try: + n.notify(title=title, message=message, priority=priority) + except Exception as exc: + log.exception("notifier %r crashed: %s", type(n).__name__, exc) + + +class NullNotifier: + """No-op notifier: used to silence a muted container's alerts for one + _check_one call. Mirrors the notify() signature of the real notifiers.""" + + def notify(self, title: str, message: str, priority: int | None = None) -> None: + pass diff --git a/src/inspector/portainer_client.py b/src/inspector/portainer_client.py new file mode 100755 index 0000000..befd1fd --- /dev/null +++ b/src/inspector/portainer_client.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass + +import requests + +from .config import PortainerConfig, PortainerEnvironmentConfig + + +log = logging.getLogger("inspector.portainer") + + +@dataclass(frozen=True) +class PortainerRedeployResult: + ok: bool + message: str = "" + + +class PortainerClient: + """Talks to one or more named Portainer environments to redeploy stacks. + + Always safe to construct and call: never raises. Any error (missing + environment, missing token env var, network failure, stack not found, + unexpected response) yields PortainerRedeployResult(ok=False, message=...). + + Security: the API token value is read from the OS environment per-call + and is never logged, returned, or included in any message. Only the + *name* of the environment variable and the fact that "a Portainer API + call was attempted" are logged. + """ + + def __init__(self, config: PortainerConfig, timeout: float = 15.0): + self._config = config + self._timeout = timeout + + @property + def enabled(self) -> bool: + return self._config.enabled + + def _resolve_environment(self, environment_name: str) -> PortainerEnvironmentConfig | None: + return self._config.environments.get(environment_name) + + def _load_token(self, env_cfg: PortainerEnvironmentConfig) -> str | None: + if env_cfg.api_token: + return env_cfg.api_token + if env_cfg.api_token_env: + return os.environ.get(env_cfg.api_token_env) + return None + + def redeploy_stack(self, environment_name: str, stack_name: str) -> PortainerRedeployResult: + env_cfg = self._resolve_environment(environment_name) + if env_cfg is None: + return PortainerRedeployResult( + ok=False, + message=( + f"Portainer environment {environment_name!r} is not configured " + f"under integrations.portainer.environments" + ), + ) + + if not env_cfg.base_url: + return PortainerRedeployResult( + ok=False, + message=f"Portainer environment {environment_name!r} has no base_url configured", + ) + + token = self._load_token(env_cfg) + if not token: + if env_cfg.api_token_env: + msg = ( + f"Portainer API token not found in environment variable " + f"{env_cfg.api_token_env!r} for environment {environment_name!r}. " + f"Ensure the env var is set in the Inspector container, or use " + f"'api_token' to embed the token directly in config.yml." + ) + else: + msg = ( + f"Portainer environment {environment_name!r} has no API token configured. " + f"Set 'api_token_env' (name of an env var) or 'api_token' (direct value) " + f"under integrations.portainer.environments.{environment_name}." + ) + return PortainerRedeployResult(ok=False, message=msg) + + headers = {"X-API-Key": token} + verify = env_cfg.verify_ssl + + log.info( + "Portainer API call attempted: environment=%s endpoint_id=%s stack=%s", + environment_name, env_cfg.endpoint_id, stack_name, + ) + + try: + stack = self._find_stack(env_cfg, headers, verify, stack_name) + if stack is None: + log.warning( + "stack %r not found in Portainer environment %r (endpoint_id=%s)", + stack_name, environment_name, env_cfg.endpoint_id, + ) + return PortainerRedeployResult( + ok=False, + message=( + f"Stack {stack_name!r} not found in Portainer environment " + f"{environment_name!r} (endpoint_id={env_cfg.endpoint_id})" + ), + ) + + stack_id = stack["Id"] + log.debug("found stack %r with Id=%s", stack_name, stack_id) + file_content = self._fetch_stack_file(env_cfg, headers, verify, stack_id) + env_vars = stack.get("Env") or [] + + log.debug("sending PUT /api/stacks/%s for stack %r", stack_id, stack_name) + resp = requests.put( + f"{env_cfg.base_url}/api/stacks/{stack_id}", + params={"endpointId": env_cfg.endpoint_id}, + headers=headers, + json={ + "stackFileContent": file_content, + "env": env_vars, + "prune": False, + "pullImage": True, + }, + timeout=self._timeout, + verify=verify, + ) + resp.raise_for_status() + except requests.HTTPError as exc: + body = "" + try: + body = exc.response.text[:500] + except Exception: + pass + detail = f"HTTP {exc.response.status_code}: {body}" if body else str(exc) + log.error( + "Portainer stack redeploy failed: environment=%s stack=%s error=%s", + environment_name, stack_name, detail, + ) + return PortainerRedeployResult(ok=False, message=f"Portainer API error: {detail}") + except requests.RequestException as exc: + log.error( + "Portainer stack redeploy failed: environment=%s stack=%s error=%s", + environment_name, stack_name, exc, + ) + return PortainerRedeployResult(ok=False, message=f"Portainer API request failed: {exc}") + except Exception as exc: + log.exception( + "Portainer stack redeploy crashed: environment=%s stack=%s", + environment_name, stack_name, + ) + return PortainerRedeployResult(ok=False, message=f"unexpected error: {exc}") + + log.info( + "Portainer stack redeploy succeeded: environment=%s stack=%s stack_id=%s", + environment_name, stack_name, stack_id, + ) + return PortainerRedeployResult(ok=True, message=f"redeployed stack {stack_name!r}") + + def _find_stack(self, env_cfg, headers, verify, stack_name): + resp = requests.get( + f"{env_cfg.base_url}/api/stacks", headers=headers, timeout=self._timeout, verify=verify, + ) + resp.raise_for_status() + stacks = resp.json() or [] + log.debug( + "_find_stack: %d stacks total; searching name=%r endpoint_id=%s", + len(stacks), stack_name, env_cfg.endpoint_id, + ) + for stack in stacks: + if stack.get("Name") == stack_name and stack.get("EndpointId") == env_cfg.endpoint_id: + return stack + return None + + def _fetch_stack_file(self, env_cfg, headers, verify, stack_id) -> str: + log.debug("fetching compose file for stack_id=%s", stack_id) + resp = requests.get( + f"{env_cfg.base_url}/api/stacks/{stack_id}/file", + headers=headers, timeout=self._timeout, verify=verify, + ) + resp.raise_for_status() + return (resp.json() or {}).get("StackFileContent", "") diff --git a/src/inspector/recovery.py b/src/inspector/recovery.py new file mode 100755 index 0000000..a66f58b --- /dev/null +++ b/src/inspector/recovery.py @@ -0,0 +1,544 @@ +from __future__ import annotations + +import logging +import time +from datetime import datetime, timedelta + +from .config import AppConfig, ContainerConfig, PortainerConfig +from .docker_client import restart_container, start_container +from .eventstore import ( + EVENT_PORTAINER_REDEPLOY, + EVENT_PORTAINER_REDEPLOY_FAILED, + EVENT_PORTAINER_REDEPLOY_SKIPPED, + EVENT_RESTART, + EVENT_RESTART_DRYRUN, + EVENT_RESTART_FAILED, + EventStore, +) +from .formatting import format_timestamp +from .health import HealthResult +from .http_check import check_http +from .notifier import GotifyNotifier +from .portainer_client import PortainerClient +from .state import ContainerState + + +log = logging.getLogger("inspector.recovery") + +_STALE_NAMESPACE_MARKERS = ( + "joining network namespace of container", + "no such container", +) + + +def is_stale_vpn_namespace_error(message: str | None) -> bool: + """Case-insensitive substring match for the Docker error text emitted + when a VPN-routed container's `network_mode: container:` target no + longer exists (the VPN container was recreated).""" + if not message: + return False + lowered = message.lower() + return any(marker in lowered for marker in _STALE_NAMESPACE_MARKERS) + + +def resolve_portainer_environment( + container_config: ContainerConfig, + portainer_config: PortainerConfig, +) -> tuple[str | None, str | None]: + """Resolve the Portainer environment for this container's redeploy. + + Returns (environment_name, error); exactly one is non-None. + """ + name = container_config.name + + if container_config.portainer_environment: + return container_config.portainer_environment, None + + if portainer_config.default_environment: + return portainer_config.default_environment, None + + configured = list(portainer_config.environments.keys()) + if len(configured) == 1: + only_env = configured[0] + log.info( + "resolved Portainer environment %s for %s (only environment configured)", + only_env, name, + ) + return only_env, None + + return None, ( + f"{name} requests recovery_action 'portainer_stack_redeploy' but " + f"integrations.portainer.default_environment is not set and " + f"{len(configured)} environments are configured " + f"({sorted(configured)}); cannot resolve which Portainer environment to use" + ) + + +def stack_cooldown_key(environment: str, stack_name: str) -> str: + return f"{environment}:{stack_name}" + + +def in_stack_cooldown(stack_cooldowns: dict[str, datetime], key: str, now: datetime) -> bool: + until = stack_cooldowns.get(key) + return until is not None and now < until + + +def perform_recovery( + *, + app: AppConfig, + container_config: ContainerConfig, + state: ContainerState, + result: HealthResult, + notifier: GotifyNotifier, + eventstore: EventStore, + portainer_client: PortainerClient, + stack_cooldowns: dict[str, datetime], + client, + dry_run: bool, + width: int, + now: datetime, +) -> None: + action = container_config.recovery_action or "restart_container" + + if action == "portainer_stack_redeploy": + _perform_portainer_redeploy( + app=app, + container_config=container_config, + state=state, + result=result, + notifier=notifier, + eventstore=eventstore, + portainer_client=portainer_client, + stack_cooldowns=stack_cooldowns, + dry_run=dry_run, + width=width, + now=now, + ) + return + + if action == "restart_then_redeploy": + _perform_restart_then_redeploy( + app=app, + container_config=container_config, + state=state, + result=result, + notifier=notifier, + eventstore=eventstore, + portainer_client=portainer_client, + stack_cooldowns=stack_cooldowns, + client=client, + dry_run=dry_run, + width=width, + now=now, + ) + return + + _perform_docker_action( + action=action, + container_config=container_config, + state=state, + result=result, + notifier=notifier, + eventstore=eventstore, + client=client, + dry_run=dry_run, + width=width, + now=now, + ) + + +def _perform_restart_then_redeploy( + *, + app: AppConfig, + container_config: ContainerConfig, + state: ContainerState, + result: HealthResult, + notifier: GotifyNotifier, + eventstore: EventStore, + portainer_client: PortainerClient, + stack_cooldowns: dict[str, datetime], + client, + dry_run: bool, + width: int, + now: datetime, +) -> None: + """Tiered recovery: try `docker restart` first, escalate to a Portainer stack + redeploy only when needed. + + Tier 1 (first failure episode): attempt a plain `docker restart`. A VPN-routed + container whose sidecar was merely *restarted* (same container id) re-establishes + its network this way, avoiding a full redeploy. + + Escalate to redeploy when either: + - the restart command itself errors (e.g. the sidecar was *recreated*, so the + stored `network_mode: container:` is stale -> "No such container"), or + - the container is still failing on a later recovery cycle + (`state.restarted_this_episode` is already set), meaning the restart didn't stick. + + The episode flag resets when the container is next seen HEALTHY (see main._check_one). + """ + name = container_config.name + + def redeploy() -> None: + _perform_portainer_redeploy( + app=app, + container_config=container_config, + state=state, + result=result, + notifier=notifier, + eventstore=eventstore, + portainer_client=portainer_client, + stack_cooldowns=stack_cooldowns, + dry_run=dry_run, + width=width, + now=now, + ) + + # Already restarted this episode and still failing -> escalate to redeploy. + if state.restarted_this_episode: + log.warning( + "%-*s still failing after a restart; escalating to Portainer stack redeploy", + width, name, + ) + redeploy() + return + + if dry_run: + log.warning("%-*s DRY RUN: would restart (tier 1 of restart_then_redeploy)", width, name) + state.record_restart(now, container_config.cooldown_seconds) + state.restarted_this_episode = True + eventstore.record( + container=name, + event_type=EVENT_RESTART_DRYRUN, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"would restart due to {result.status} (tier 1, then redeploy if still failing)", + ) + return + + log.warning( + "%-*s restarting (tier 1) due to %d consecutive %s checks", + width, name, state.error_count, result.status, + ) + notifier.notify( + title=f"🔁 Inspector: restarting {name}", + message=( + f"Restarting **{name}** after **{state.error_count}** consecutive " + f"**{result.status}** check(s).\n\n" + f"- If it doesn't recover, Inspector escalates to a Portainer stack redeploy\n" + f"- Cooldown: **{container_config.cooldown_seconds}s** after restart\n" + f"- Time: {format_timestamp(now)}" + ), + priority=6, + ) + + success, err = restart_container(client, name) + + if success: + state.record_restart(now, container_config.cooldown_seconds) + state.restarted_this_episode = True + state.last_notification_state = "RESTARTED" + state.restart_limit_notified = False + log.info( + "%-*s restart succeeded; cooldown until %s (will redeploy if still failing)", + width, name, state.cooldown_until.isoformat(), + ) + eventstore.record( + container=name, + event_type=EVENT_RESTART, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"restart succeeded; cooldown until {state.cooldown_until.isoformat()}", + ) + notifier.notify( + title=f"✅ Inspector: {name} restart completed", + message=( + f"**{name}** was successfully restarted by Docker.\n\n" + f"- Previous status: *{result.status}*\n" + f"- Cooldown until: {state.cooldown_until.isoformat()}\n" + f"- Time: {format_timestamp(now)}" + ), + priority=3, + ) + return + + # Restart failed -> escalate to redeploy this same round. + log.error("%-*s restart failed: %s; escalating to Portainer stack redeploy", width, name, err) + if is_stale_vpn_namespace_error(err): + log.info( + "%-*s restart failure is a stale VPN namespace error; redeploy is required", + width, name, + ) + eventstore.record( + container=name, + event_type=EVENT_RESTART_FAILED, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"restart failed ({err}); escalating to redeploy", + ) + redeploy() + + +def _perform_docker_action( + *, + action: str, + container_config: ContainerConfig, + state: ContainerState, + result: HealthResult, + notifier: GotifyNotifier, + eventstore: EventStore, + client, + dry_run: bool, + width: int, + now: datetime, +) -> None: + name = container_config.name + verb = "restart" if action == "restart_container" else "start" + + if dry_run: + log.warning("%-*s DRY RUN: would %s due to %s status", width, name, verb, result.status) + state.record_restart(now, container_config.cooldown_seconds) + eventstore.record( + container=name, + event_type=EVENT_RESTART_DRYRUN, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"would {verb} due to {result.status}", + ) + return + + log.warning( + "%-*s %sing due to %d consecutive %s checks", + width, name, verb.capitalize(), state.error_count, result.status, + ) + notifier.notify( + title=f"🔁 Inspector: {verb}ing {name}", + message=( + f"{verb.capitalize()}ing **{name}** after **{state.error_count}** consecutive " + f"**{result.status}** check(s).\n\n" + f"- Cooldown: **{container_config.cooldown_seconds}s** after {verb}\n" + f"- Time: {format_timestamp(now)}" + ), + priority=6, + ) + + if action == "restart_container": + success, err = restart_container(client, name) + else: + success, err = start_container(client, name) + + if success: + state.record_restart(now, container_config.cooldown_seconds) + state.last_notification_state = "RESTARTED" + state.restart_limit_notified = False + log.info("%-*s %s succeeded; cooldown until %s", width, name, verb, state.cooldown_until.isoformat()) + eventstore.record( + container=name, + event_type=EVENT_RESTART, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"{verb} succeeded; cooldown until {state.cooldown_until.isoformat()}", + ) + notifier.notify( + title=f"✅ Inspector: {name} {verb} completed", + message=( + f"**{name}** was successfully {verb}ed by Docker.\n\n" + f"- Previous status: *{result.status}*\n" + f"- Cooldown until: {state.cooldown_until.isoformat()}\n" + f"- Time: {format_timestamp(now)}" + ), + priority=3, + ) + else: + log.error("%-*s %s failed: %s", width, name, verb, err) + notifier.notify( + title=f"🛑 Inspector: {name} {verb} FAILED", + message=( + f"{verb.capitalize()} of **{name}** failed.\n\n" + f"- Status: **{result.status}**\n" + f"- Error count: **{state.error_count}**\n" + f"- Time: {format_timestamp(now)}\n\n" + f"```\n{err}\n```" + ), + priority=9, + ) + eventstore.record( + container=name, + event_type=EVENT_RESTART_FAILED, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=str(err), + ) + + +def _perform_portainer_redeploy( + *, + app: AppConfig, + container_config: ContainerConfig, + state: ContainerState, + result: HealthResult, + notifier: GotifyNotifier, + eventstore: EventStore, + portainer_client: PortainerClient, + stack_cooldowns: dict[str, datetime], + dry_run: bool, + width: int, + now: datetime, +) -> None: + name = container_config.name + portainer_config = app.integrations.portainer + stack_name = container_config.portainer_stack_name + + if is_stale_vpn_namespace_error(result.message): + log.info("%-*s detected stale VPN network namespace error", width, name) + + if not portainer_config.enabled: + log.warning( + "%-*s requests recovery_action 'portainer_stack_redeploy' but " + "integrations.portainer.enabled is false; skipping", + width, name, + ) + eventstore.record( + container=name, + event_type=EVENT_PORTAINER_REDEPLOY_SKIPPED, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message="integrations.portainer.enabled is false", + ) + state.record_restart(now, container_config.cooldown_seconds) + return + + environment, error = resolve_portainer_environment(container_config, portainer_config) + if error: + log.error("%-*s %s", width, name, error) + eventstore.record( + container=name, + event_type=EVENT_PORTAINER_REDEPLOY_SKIPPED, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=error, + ) + state.record_restart(now, container_config.cooldown_seconds) + return + + key = stack_cooldown_key(environment, stack_name) + + if in_stack_cooldown(stack_cooldowns, key, now): + log.info("Portainer stack redeploy skipped due to cooldown for %s", key) + eventstore.record( + container=name, + event_type=EVENT_PORTAINER_REDEPLOY_SKIPPED, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"cooldown active for {key}", + ) + state.record_restart(now, container_config.cooldown_seconds) + return + + wait_seconds = portainer_config.post_redeploy_wait_seconds + + if dry_run: + log.warning("%-*s DRY RUN: would resolve Portainer environment %s", width, name, environment) + log.warning("%-*s DRY RUN: would redeploy Portainer stack %s", width, name, key) + log.warning("%-*s DRY RUN: would wait %d seconds after stack redeploy", width, name, wait_seconds) + state.record_restart(now, container_config.cooldown_seconds) + eventstore.record( + container=name, + event_type=EVENT_RESTART_DRYRUN, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"would redeploy Portainer stack {key}", + ) + return + + log.info("resolved Portainer environment %s for %s", environment, name) + log.warning("triggering Portainer stack redeploy for environment %s stack %s", environment, stack_name) + notifier.notify( + title=f"🔁 Inspector: redeploying Portainer stack {stack_name}", + message=( + f"Redeploying Portainer stack **{stack_name}** (environment **{environment}**) " + f"to recover **{name}**.\n\n" + f"- Status: **{result.status}**\n" + f"- Time: {format_timestamp(now)}" + ), + priority=6, + ) + + redeploy_result = portainer_client.redeploy_stack(environment, stack_name) + + if not redeploy_result.ok: + log.error("%-*s Portainer stack redeploy failed for %s: %s", width, name, key, redeploy_result.message) + notifier.notify( + title=f"🛑 Inspector: Portainer redeploy FAILED for {stack_name}", + message=( + f"Portainer redeploy of **{stack_name}** (environment **{environment}**) failed.\n\n" + f"- Triggered by: **{name}** ({result.status})\n" + f"- Time: {format_timestamp(now)}\n\n" + f"```\n{redeploy_result.message}\n```" + ), + priority=9, + ) + eventstore.record( + container=name, + event_type=EVENT_PORTAINER_REDEPLOY_FAILED, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=redeploy_result.message, + ) + state.record_restart(now, container_config.cooldown_seconds) + return + + stack_cooldowns[key] = now + timedelta(seconds=portainer_config.stack_redeploy_cooldown_seconds) + log.info("%-*s Portainer stack redeploy succeeded for %s", width, name, key) + eventstore.record( + container=name, + event_type=EVENT_PORTAINER_REDEPLOY, + old_status=state.previous_status, + new_status=result.status, + error_count=state.error_count, + message=f"redeployed stack {key}", + ) + notifier.notify( + title=f"✅ Inspector: Portainer stack {stack_name} redeployed", + message=( + f"Portainer stack **{stack_name}** (environment **{environment}**) was " + f"successfully redeployed to recover **{name}**.\n\n" + f"- Triggered by: *{result.status}*\n" + f"- Post-redeploy wait: {wait_seconds}s\n" + f"- Time: {format_timestamp(now)}" + ), + priority=4, + ) + + log.info("waiting %d seconds after stack redeploy", wait_seconds) + time.sleep(wait_seconds) + + if container_config.http_check.enabled: + recheck = check_http(container_config.http_check) + if recheck.ok: + log.info("recheck succeeded for %s", name) + else: + log.warning("recheck failed for %s: %s", name, recheck.message) + notifier.notify( + title=f"⚠️ Inspector: {name} recheck failed after redeploy", + message=( + f"Portainer stack **{stack_name}** was redeployed, but the HTTP recheck " + f"for **{name}** still reports a failure.\n\n" + f"- Recheck result: *{recheck.message}*\n" + f"- Time: {format_timestamp(now)}" + ), + priority=6, + ) + + state.record_restart(now, container_config.cooldown_seconds) diff --git a/src/inspector/state.py b/src/inspector/state.py new file mode 100755 index 0000000..86a002e --- /dev/null +++ b/src/inspector/state.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from .config import ContainerConfig +from .health import NO_HEALTH_CHECK, STARTING + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def localnow() -> datetime: + return datetime.now() + + +@dataclass +class ContainerState: + name: str + current_status: str | None = None + previous_status: str | None = None + error_count: int = 0 + cooldown_until: datetime | None = None + last_restart_at: datetime | None = None + restart_history: deque[datetime] = field(default_factory=lambda: deque(maxlen=64)) + last_notification_state: str | None = None + restart_limit_notified: bool = False + restarted_this_episode: bool = False + last_check_at: datetime | None = None + + def in_cooldown(self, now: datetime) -> bool: + return self.cooldown_until is not None and now < self.cooldown_until + + def in_grace(self, now: datetime, grace_seconds: int) -> bool: + if self.last_restart_at is None or grace_seconds <= 0: + return False + return now < self.last_restart_at + timedelta(seconds=grace_seconds) + + def restarts_last_hour(self, now: datetime) -> int: + cutoff = now - timedelta(hours=1) + return sum(1 for ts in self.restart_history if ts >= cutoff) + + def record_restart(self, now: datetime, cooldown_seconds: int) -> None: + self.last_restart_at = now + self.cooldown_until = now + timedelta(seconds=cooldown_seconds) + self.restart_history.append(now) + self.error_count = 0 + + +def should_count_failure( + status: str, + container_config: ContainerConfig, + in_grace: bool, +) -> bool: + if status == STARTING and STARTING not in container_config.restart_on: + return False + if status == NO_HEALTH_CHECK and NO_HEALTH_CHECK not in container_config.restart_on: + return False + if status not in container_config.restart_on: + return False + if in_grace: + return False + return True + + +def should_restart( + state: ContainerState, + container_config: ContainerConfig, + now: datetime, +) -> tuple[bool, str | None]: + if state.in_cooldown(now): + return False, "in cooldown" + if state.error_count < container_config.max_error_count: + return False, "below threshold" + if state.restarts_last_hour(now) >= container_config.max_restarts_per_hour: + return False, "hourly restart limit reached" + return True, None diff --git a/src/inspector/uptime_kuma.py b/src/inspector/uptime_kuma.py new file mode 100755 index 0000000..fb82546 --- /dev/null +++ b/src/inspector/uptime_kuma.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass + +import requests + +from .config import UptimeKumaConfig + +log = logging.getLogger("inspector.uptime_kuma") + + +@dataclass(frozen=True) +class HeartbeatSnapshot: + up_by_monitor: dict[str, bool] + + def is_up(self, monitor_name: str) -> bool | None: + return self.up_by_monitor.get(monitor_name) + + +_EMPTY = HeartbeatSnapshot(up_by_monitor={}) + + +class UptimeKumaClient: + """Polls a public Uptime Kuma status-page heartbeat endpoint. + + Always safe to construct and call: disabled config, network errors, or + an unexpected response shape all yield an empty snapshot (every + `is_up()` returns None, meaning "no signal") rather than raising. + """ + + def __init__(self, config: UptimeKumaConfig, timeout: float = 5.0): + self._config = config + self._timeout = timeout + + @property + def enabled(self) -> bool: + return ( + self._config.enabled + and bool(self._config.url) + and bool(self._config.status_page_slug) + ) + + def fetch(self) -> HeartbeatSnapshot: + if not self.enabled: + return _EMPTY + + url = f"{self._config.url}/api/status-page/heartbeat/{self._config.status_page_slug}" + try: + resp = requests.get(url, timeout=self._timeout) + resp.raise_for_status() + data = resp.json() + return self._parse(data) + except Exception as exc: + log.warning("Uptime Kuma heartbeat fetch failed (%s): %s", url, exc) + return _EMPTY + + def _parse(self, data: dict) -> HeartbeatSnapshot: + # Expected shape: + # {"publicGroupList": [{"monitorList": [{"id": 1, "name": "..."}]}], + # "heartbeatList": {"1": [{"status": 1, ...}, ...]}} # 1 == up + up_by_monitor: dict[str, bool] = {} + id_to_name: dict[int, str] = {} + for group in data.get("publicGroupList") or []: + for monitor in group.get("monitorList") or []: + mid, mname = monitor.get("id"), monitor.get("name") + if mid is not None and mname: + id_to_name[mid] = mname + + heartbeat_list = data.get("heartbeatList") or {} + for mid, name in id_to_name.items(): + beats = heartbeat_list.get(str(mid)) or heartbeat_list.get(mid) + if beats: + up_by_monitor[name] = beats[-1].get("status") == 1 + + return HeartbeatSnapshot(up_by_monitor=up_by_monitor)