Inspector 0.9.4: containers without a health check recover properly

This commit is contained in:
Europium 63 committed 2026-10-28 07:21:59 -07:00
1 parent 57100d0a7b
commit 6aed0cf9a7
4 files changed
+49 -9

No files matched your search

+20
View File
@@ -7,6 +7,26 @@ Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html
--- ---
## [0.9.4] — 2026-10-28
### Fixed
- **Containers without a Docker health check now "recover" properly.** Many images
don't define a health check, so Docker reports them as `NO HEALTH CHECK` even when
they work. Inspector only treated `HEALTHY` as back to normal, so for these
containers:
- the error count never reset, and one-off failures days apart added up to a restart;
- no "recovered" alert was sent;
- with *Restart, then redeploy*, the next unrelated problem skipped the restart and
redeployed the whole stack.
Now `NO HEALTH CHECK` counts as working (unless you made it a restart trigger): the
count resets, the "recovered" alert is sent ("back to running (no health check)"),
and the next problem starts with a plain restart. Containers with a health check
behave exactly as before.
---
## [0.9.3] — 2026-10-28 ## [0.9.3] — 2026-10-28
### Changed ### Changed
+1 -1
View File
@@ -1 +1 @@
__version__ = "0.9.3" __version__ = "0.9.4"
+26 -7
View File
@@ -63,6 +63,19 @@ def _request_stop(signum, _frame):
_stop = True _stop = True
def _is_normal(status: str | None, container_config: ContainerConfig) -> bool:
"""True when `status` means the container is working.
HEALTHY always counts. NO HEALTH CHECK (running, but the image defines no
health check) counts too, unless the user made it a restart trigger.
Many images have no health check, and without this their error count and
restart episode would never reset.
"""
if status == HEALTHY:
return True
return status == NO_HEALTH_CHECK and NO_HEALTH_CHECK not in container_config.restart_on
def _apply_reachability_override( def _apply_reachability_override(
container_config: ContainerConfig, container_config: ContainerConfig,
result: HealthResult, result: HealthResult,
@@ -158,17 +171,23 @@ def _check_one(
message=result.message, message=result.message,
) )
is_normal = _is_normal(state.current_status, container_config)
if ( if (
state.previous_status is not None state.previous_status is not None
and state.previous_status != HEALTHY and not _is_normal(state.previous_status, container_config)
and state.current_status == HEALTHY and is_normal
# HEALTHY here means "a recovery was already announced", for either status.
and state.last_notification_state != HEALTHY and state.last_notification_state != HEALTHY
): ):
log.info("%-*s recovered: %s -> HEALTHY", width, name, state.previous_status) log.info("%-*s recovered: %s -> %s", width, name, state.previous_status, state.current_status)
back_to = (
"**HEALTHY**" if state.current_status == HEALTHY
else "**running** (no health check)"
)
notifier.notify( notifier.notify(
title=f"✅ Inspector: {name} recovered", title=f"✅ Inspector: {name} recovered",
message=( message=(
f"**{name}** is back to **HEALTHY** (was *{state.previous_status}*).\n\n" f"**{name}** is back to {back_to} (was *{state.previous_status}*).\n\n"
f"- Errors before recovery: **{state.error_count}**\n" f"- Errors before recovery: **{state.error_count}**\n"
f"- Time: {format_timestamp(now, with_zone=True)}" f"- Time: {format_timestamp(now, with_zone=True)}"
), ),
@@ -178,7 +197,7 @@ def _check_one(
container=name, container=name,
event_type=EVENT_RECOVERY, event_type=EVENT_RECOVERY,
old_status=state.previous_status, old_status=state.previous_status,
new_status=HEALTHY, new_status=state.current_status,
error_count=state.error_count, error_count=state.error_count,
) )
state.last_notification_state = HEALTHY state.last_notification_state = HEALTHY
@@ -253,9 +272,9 @@ def _check_one(
message=result.message, message=result.message,
) )
state.last_notification_state = result.status state.last_notification_state = result.status
elif result.status == HEALTHY: elif is_normal:
if state.error_count != 0: if state.error_count != 0:
log.info("%-*s healthy again; resetting error count", width, name) log.info("%-*s back to normal; resetting error count", width, name)
state.error_count = 0 state.error_count = 0
state.restarted_this_episode = False state.restarted_this_episode = False
+2 -1
View File
@@ -175,7 +175,8 @@ def _perform_restart_then_redeploy(
- the container is still failing on a later recovery cycle - the container is still failing on a later recovery cycle
(`state.restarted_this_episode` is already set), meaning the restart didn't stick. (`state.restarted_this_episode` is already set), meaning the restart didn't stick.
The episode flag resets when the container is next seen HEALTHY (see main._check_one). The episode flag resets when the container is next seen working: HEALTHY, or
NO HEALTH CHECK when that isn't a restart trigger (see main._is_normal).
""" """
name = container_config.name name = container_config.name