From 2a9c57b7d71f5e99ea21863d3dbfc46e3002bfcb Mon Sep 17 00:00:00 2001 From: Europium 63 Date: Wed, 28 Oct 2026 07:28:45 -0700 Subject: [PATCH] Inspector 0.9.5: stack redeploy no longer blocks other checks --- CHANGELOG.md | 15 +++++++ src/inspector/__init__.py | 2 +- src/inspector/assets/guide/guide.html | 5 ++- src/inspector/main.py | 7 ++- src/inspector/recovery.py | 61 ++++++++++++++++++--------- src/inspector/state.py | 2 + 6 files changed, 68 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 804bf87..2eebb24 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html --- +## [0.9.5] — 2026-10-28 + +### Fixed + +- **A stack redeploy no longer freezes Inspector.** After a successful Portainer + redeploy, Inspector used to stop everything for `post_redeploy_wait_seconds` + (default 90 s): no other container was checked, dashboard times stood still, and + stopping Inspector during the wait wasn't clean. Now the redeployed app is left to + settle while every other container keeps being checked, and its one-time HTTP + re-check (with the "recheck failed" alert) happens on its first check after the wait. +- The redeployed app's cooldown now starts after the settle time. Before, the wait + ate into it (with the defaults, 300 s cooldown became 210 s). + +--- + ## [0.9.4] — 2026-10-28 ### Fixed diff --git a/src/inspector/__init__.py b/src/inspector/__init__.py index e94731c..f8c6ac7 100755 --- a/src/inspector/__init__.py +++ b/src/inspector/__init__.py @@ -1 +1 @@ -__version__ = "0.9.4" +__version__ = "0.9.5" diff --git a/src/inspector/assets/guide/guide.html b/src/inspector/assets/guide/guide.html index 14ac34b..b55e7f8 100755 --- a/src/inspector/assets/guide/guide.html +++ b/src/inspector/assets/guide/guide.html @@ -356,8 +356,9 @@ folder (or a backup of it) can read the token.

stack_redeploy_cooldown_secondsMinimum time between two redeploys of the same stack (default 300). Several apps often share a stack; this stops them triggering a redeploy each. - post_redeploy_wait_secondsHow long Inspector waits after a redeploy for the - stack to come up before judging it again (default 90). + post_redeploy_wait_secondsHow long Inspector gives the stack to come up after + a redeploy before judging that app again (default 90). Everything else keeps being checked + meanwhile.

If Test connection fails

diff --git a/src/inspector/main.py b/src/inspector/main.py index 5d5c34e..baa07db 100755 --- a/src/inspector/main.py +++ b/src/inspector/main.py @@ -41,7 +41,7 @@ from .maintenance import MaintenanceTracker from .notification_mute import NotificationMuteTracker from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier from .portainer_client import PortainerClient -from .recovery import perform_recovery +from .recovery import perform_recovery, run_post_redeploy_recheck from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient from .state import ( ContainerState, @@ -153,6 +153,11 @@ def _check_one( if notification_mute.is_muted(name): notifier = NullNotifier() + if state.redeploy_recheck_at is not None and now >= state.redeploy_recheck_at: + run_post_redeploy_recheck( + container_config=container_config, state=state, notifier=notifier, now=now, + ) + if state.previous_status != state.current_status: log.info( "%-*s status: %s -> %s%s", diff --git a/src/inspector/recovery.py b/src/inspector/recovery.py index 9662367..9281fbf 100755 --- a/src/inspector/recovery.py +++ b/src/inspector/recovery.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import time from datetime import datetime, timedelta from .config import AppConfig, ContainerConfig, PortainerConfig @@ -522,24 +521,46 @@ def _perform_portainer_redeploy( priority=4, ) - log.info("waiting %d seconds after stack redeploy", wait_seconds) - time.sleep(wait_seconds) - + # Don't block the loop while the stack settles: keep this container in + # cooldown for the wait plus its normal cooldown, and do the one-time HTTP + # re-check on its first regular check after the wait (main._check_one). + log.info( + "%-*s letting stack %s settle for %ds; other containers keep being checked", + width, name, key, wait_seconds, + ) + state.record_restart(now, wait_seconds + container_config.cooldown_seconds) if container_config.http_check.enabled: - recheck = check_http(container_config.http_check) - if recheck.ok: - log.info("recheck succeeded for %s", name) - else: - log.warning("recheck failed for %s: %s", name, recheck.message) - notifier.notify( - title=f"⚠️ Inspector: {name} recheck failed after redeploy", - message=( - f"Portainer stack **{stack_name}** was redeployed, but the HTTP recheck " - f"for **{name}** still reports a failure.\n\n" - f"- Recheck result: *{recheck.message}*\n" - f"- Time: {format_timestamp(now, with_zone=True)}" - ), - priority=6, - ) + state.redeploy_recheck_at = now + timedelta(seconds=wait_seconds) - state.record_restart(now, container_config.cooldown_seconds) + +def run_post_redeploy_recheck( + *, + container_config: ContainerConfig, + state: ContainerState, + notifier: GotifyNotifier, + now: datetime, +) -> None: + """One HTTP re-check after a stack redeploy has had time to settle. + + Called by main._check_one once `state.redeploy_recheck_at` has passed; + alerts if the app still doesn't answer. + """ + state.redeploy_recheck_at = None + name = container_config.name + if not container_config.http_check.enabled: + return + recheck = check_http(container_config.http_check) + if recheck.ok: + log.info("recheck succeeded for %s", name) + return + log.warning("recheck failed for %s: %s", name, recheck.message) + notifier.notify( + title=f"⚠️ Inspector: {name} recheck failed after redeploy", + message=( + f"Portainer stack **{container_config.portainer_stack_name}** was redeployed, but the HTTP recheck " + f"for **{name}** still reports a failure.\n\n" + f"- Recheck result: *{recheck.message}*\n" + f"- Time: {format_timestamp(now, with_zone=True)}" + ), + priority=6, + ) diff --git a/src/inspector/state.py b/src/inspector/state.py index 86a002e..6450f97 100755 --- a/src/inspector/state.py +++ b/src/inspector/state.py @@ -29,6 +29,8 @@ class ContainerState: restart_limit_notified: bool = False restarted_this_episode: bool = False last_check_at: datetime | None = None + # After a stack redeploy: when to do the one-time HTTP re-check. + redeploy_recheck_at: datetime | None = None def in_cooldown(self, now: datetime) -> bool: return self.cooldown_until is not None and now < self.cooldown_until