Inspector 0.9.5: stack redeploy no longer blocks other checks

This commit is contained in:
Europium 63 committed 2026-10-28 07:28:45 -07:00
1 parent 6aed0cf9a7
commit 3bd584c6d1
6 files changed
+68 -24

No files matched your search

+15
View File
@@ -7,6 +7,21 @@ Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html
---
## [0.9.5] — 2026-10-28
### Fixed
- **A stack redeploy no longer freezes Inspector.** After a successful Portainer
redeploy, Inspector used to stop everything for `post_redeploy_wait_seconds`
(default 90 s): no other container was checked, dashboard times stood still, and
stopping Inspector during the wait wasn't clean. Now the redeployed app is left to
settle while every other container keeps being checked, and its one-time HTTP
re-check (with the "recheck failed" alert) happens on its first check after the wait.
- The redeployed app's cooldown now starts after the settle time. Before, the wait
ate into it (with the defaults, 300 s cooldown became 210 s).
---
## [0.9.4] — 2026-10-28
### Fixed
+1 -1
View File
@@ -1 +1 @@
__version__ = "0.9.4"
__version__ = "0.9.5"
+3 -2
View File
@@ -356,8 +356,9 @@ folder (or a backup of it) can read the token.</p>
<tr><td>stack_redeploy_cooldown_seconds</td><td>Minimum time between two redeploys of the same
stack (default 300). Several apps often share a stack; this stops them triggering a redeploy
each.</td></tr>
<tr><td>post_redeploy_wait_seconds</td><td>How long Inspector waits after a redeploy for the
stack to come up before judging it again (default 90).</td></tr>
<tr><td>post_redeploy_wait_seconds</td><td>How long Inspector gives the stack to come up after
a redeploy before judging that app again (default 90). Everything else keeps being checked
meanwhile.</td></tr>
</table>
<h3>If Test connection fails</h3>
+6 -1
View File
@@ -41,7 +41,7 @@ from .maintenance import MaintenanceTracker
from .notification_mute import NotificationMuteTracker
from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier
from .portainer_client import PortainerClient
from .recovery import perform_recovery
from .recovery import perform_recovery, run_post_redeploy_recheck
from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient
from .state import (
ContainerState,
@@ -153,6 +153,11 @@ def _check_one(
if notification_mute.is_muted(name):
notifier = NullNotifier()
if state.redeploy_recheck_at is not None and now >= state.redeploy_recheck_at:
run_post_redeploy_recheck(
container_config=container_config, state=state, notifier=notifier, now=now,
)
if state.previous_status != state.current_status:
log.info(
"%-*s status: %s -> %s%s",
+41 -20
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import logging
import time
from datetime import datetime, timedelta
from .config import AppConfig, ContainerConfig, PortainerConfig
@@ -522,24 +521,46 @@ def _perform_portainer_redeploy(
priority=4,
)
log.info("waiting %d seconds after stack redeploy", wait_seconds)
time.sleep(wait_seconds)
# Don't block the loop while the stack settles: keep this container in
# cooldown for the wait plus its normal cooldown, and do the one-time HTTP
# re-check on its first regular check after the wait (main._check_one).
log.info(
"%-*s letting stack %s settle for %ds; other containers keep being checked",
width, name, key, wait_seconds,
)
state.record_restart(now, wait_seconds + container_config.cooldown_seconds)
if container_config.http_check.enabled:
recheck = check_http(container_config.http_check)
if recheck.ok:
log.info("recheck succeeded for %s", name)
else:
log.warning("recheck failed for %s: %s", name, recheck.message)
notifier.notify(
title=f"⚠️ Inspector: {name} recheck failed after redeploy",
message=(
f"Portainer stack **{stack_name}** was redeployed, but the HTTP recheck "
f"for **{name}** still reports a failure.\n\n"
f"- Recheck result: *{recheck.message}*\n"
f"- Time: {format_timestamp(now, with_zone=True)}"
),
priority=6,
)
state.redeploy_recheck_at = now + timedelta(seconds=wait_seconds)
state.record_restart(now, container_config.cooldown_seconds)
def run_post_redeploy_recheck(
*,
container_config: ContainerConfig,
state: ContainerState,
notifier: GotifyNotifier,
now: datetime,
) -> None:
"""One HTTP re-check after a stack redeploy has had time to settle.
Called by main._check_one once `state.redeploy_recheck_at` has passed;
alerts if the app still doesn't answer.
"""
state.redeploy_recheck_at = None
name = container_config.name
if not container_config.http_check.enabled:
return
recheck = check_http(container_config.http_check)
if recheck.ok:
log.info("recheck succeeded for %s", name)
return
log.warning("recheck failed for %s: %s", name, recheck.message)
notifier.notify(
title=f"⚠️ Inspector: {name} recheck failed after redeploy",
message=(
f"Portainer stack **{container_config.portainer_stack_name}** was redeployed, but the HTTP recheck "
f"for **{name}** still reports a failure.\n\n"
f"- Recheck result: *{recheck.message}*\n"
f"- Time: {format_timestamp(now, with_zone=True)}"
),
priority=6,
)
+2
View File
@@ -29,6 +29,8 @@ class ContainerState:
restart_limit_notified: bool = False
restarted_this_episode: bool = False
last_check_at: datetime | None = None
# After a stack redeploy: when to do the one-time HTTP re-check.
redeploy_recheck_at: datetime | None = None
def in_cooldown(self, now: datetime) -> bool:
return self.cooldown_until is not None and now < self.cooldown_until