diff --git a/CHANGELOG.md b/CHANGELOG.md index a3397f9..804bf87 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html --- +## [0.9.4] — 2026-10-28 + +### Fixed + +- **Containers without a Docker health check now "recover" properly.** Many images + don't define a health check, so Docker reports them as `NO HEALTH CHECK` even when + they work. Inspector only treated `HEALTHY` as back to normal, so for these + containers: + - the error count never reset, and one-off failures days apart added up to a restart; + - no "recovered" alert was sent; + - with *Restart, then redeploy*, the next unrelated problem skipped the restart and + redeployed the whole stack. + + Now `NO HEALTH CHECK` counts as working (unless you made it a restart trigger): the + count resets, the "recovered" alert is sent ("back to running (no health check)"), + and the next problem starts with a plain restart. Containers with a health check + behave exactly as before. + +--- + ## [0.9.3] — 2026-10-28 ### Changed diff --git a/src/inspector/__init__.py b/src/inspector/__init__.py index c598173..e94731c 100755 --- a/src/inspector/__init__.py +++ b/src/inspector/__init__.py @@ -1 +1 @@ -__version__ = "0.9.3" +__version__ = "0.9.4" diff --git a/src/inspector/main.py b/src/inspector/main.py index 1a53dfe..5d5c34e 100755 --- a/src/inspector/main.py +++ b/src/inspector/main.py @@ -63,6 +63,19 @@ def _request_stop(signum, _frame): _stop = True +def _is_normal(status: str | None, container_config: ContainerConfig) -> bool: + """True when `status` means the container is working. + + HEALTHY always counts. NO HEALTH CHECK (running, but the image defines no + health check) counts too, unless the user made it a restart trigger. + Many images have no health check, and without this their error count and + restart episode would never reset. + """ + if status == HEALTHY: + return True + return status == NO_HEALTH_CHECK and NO_HEALTH_CHECK not in container_config.restart_on + + def _apply_reachability_override( container_config: ContainerConfig, result: HealthResult, @@ -158,17 +171,23 @@ def _check_one( message=result.message, ) + is_normal = _is_normal(state.current_status, container_config) if ( state.previous_status is not None - and state.previous_status != HEALTHY - and state.current_status == HEALTHY + and not _is_normal(state.previous_status, container_config) + and is_normal + # HEALTHY here means "a recovery was already announced", for either status. and state.last_notification_state != HEALTHY ): - log.info("%-*s recovered: %s -> HEALTHY", width, name, state.previous_status) + log.info("%-*s recovered: %s -> %s", width, name, state.previous_status, state.current_status) + back_to = ( + "**HEALTHY**" if state.current_status == HEALTHY + else "**running** (no health check)" + ) notifier.notify( title=f"✅ Inspector: {name} recovered", message=( - f"**{name}** is back to **HEALTHY** (was *{state.previous_status}*).\n\n" + f"**{name}** is back to {back_to} (was *{state.previous_status}*).\n\n" f"- Errors before recovery: **{state.error_count}**\n" f"- Time: {format_timestamp(now, with_zone=True)}" ), @@ -178,7 +197,7 @@ def _check_one( container=name, event_type=EVENT_RECOVERY, old_status=state.previous_status, - new_status=HEALTHY, + new_status=state.current_status, error_count=state.error_count, ) state.last_notification_state = HEALTHY @@ -253,9 +272,9 @@ def _check_one( message=result.message, ) state.last_notification_state = result.status - elif result.status == HEALTHY: + elif is_normal: if state.error_count != 0: - log.info("%-*s healthy again; resetting error count", width, name) + log.info("%-*s back to normal; resetting error count", width, name) state.error_count = 0 state.restarted_this_episode = False diff --git a/src/inspector/recovery.py b/src/inspector/recovery.py index 71045d0..9662367 100755 --- a/src/inspector/recovery.py +++ b/src/inspector/recovery.py @@ -175,7 +175,8 @@ def _perform_restart_then_redeploy( - the container is still failing on a later recovery cycle (`state.restarted_this_episode` is already set), meaning the restart didn't stick. - The episode flag resets when the container is next seen HEALTHY (see main._check_one). + The episode flag resets when the container is next seen working: HEALTHY, or + NO HEALTH CHECK when that isn't a restart trigger (see main._is_normal). """ name = container_config.name