1 Commits
Author SHA1 Message Date
Europium 63 2a9c57b7d7 Inspector 0.9.5: stack redeploy no longer blocks other checks 2026-10-28 07:28:45 -07:00
6 changed files with 56 additions and 12 deletions

No files matched your search

+15
View File
@@ -7,6 +7,21 @@ Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html
--- ---
## [0.9.5] — 2026-10-28
### Fixed
- **A stack redeploy no longer freezes Inspector.** After a successful Portainer
redeploy, Inspector used to stop everything for `post_redeploy_wait_seconds`
(default 90 s): no other container was checked, dashboard times stood still, and
stopping Inspector during the wait wasn't clean. Now the redeployed app is left to
settle while every other container keeps being checked, and its one-time HTTP
re-check (with the "recheck failed" alert) happens on its first check after the wait.
- The redeployed app's cooldown now starts after the settle time. Before, the wait
ate into it (with the defaults, 300 s cooldown became 210 s).
---
## [0.9.4] — 2026-10-28 ## [0.9.4] — 2026-10-28
### Fixed ### Fixed
+1 -1
View File
@@ -1 +1 @@
__version__ = "0.9.4" __version__ = "0.9.5"
+3 -2
View File
@@ -356,8 +356,9 @@ folder (or a backup of it) can read the token.</p>
<tr><td>stack_redeploy_cooldown_seconds</td><td>Minimum time between two redeploys of the same <tr><td>stack_redeploy_cooldown_seconds</td><td>Minimum time between two redeploys of the same
stack (default 300). Several apps often share a stack; this stops them triggering a redeploy stack (default 300). Several apps often share a stack; this stops them triggering a redeploy
each.</td></tr> each.</td></tr>
<tr><td>post_redeploy_wait_seconds</td><td>How long Inspector waits after a redeploy for the <tr><td>post_redeploy_wait_seconds</td><td>How long Inspector gives the stack to come up after
stack to come up before judging it again (default 90).</td></tr> a redeploy before judging that app again (default 90). Everything else keeps being checked
meanwhile.</td></tr>
</table> </table>
<h3>If Test connection fails</h3> <h3>If Test connection fails</h3>
+6 -1
View File
@@ -41,7 +41,7 @@ from .maintenance import MaintenanceTracker
from .notification_mute import NotificationMuteTracker from .notification_mute import NotificationMuteTracker
from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier
from .portainer_client import PortainerClient from .portainer_client import PortainerClient
from .recovery import perform_recovery from .recovery import perform_recovery, run_post_redeploy_recheck
from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient
from .state import ( from .state import (
ContainerState, ContainerState,
@@ -153,6 +153,11 @@ def _check_one(
if notification_mute.is_muted(name): if notification_mute.is_muted(name):
notifier = NullNotifier() notifier = NullNotifier()
if state.redeploy_recheck_at is not None and now >= state.redeploy_recheck_at:
run_post_redeploy_recheck(
container_config=container_config, state=state, notifier=notifier, now=now,
)
if state.previous_status != state.current_status: if state.previous_status != state.current_status:
log.info( log.info(
"%-*s status: %s -> %s%s", "%-*s status: %s -> %s%s",
+29 -8
View File
@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
import logging import logging
import time
from datetime import datetime, timedelta from datetime import datetime, timedelta
from .config import AppConfig, ContainerConfig, PortainerConfig from .config import AppConfig, ContainerConfig, PortainerConfig
@@ -522,24 +521,46 @@ def _perform_portainer_redeploy(
priority=4, priority=4,
) )
log.info("waiting %d seconds after stack redeploy", wait_seconds) # Don't block the loop while the stack settles: keep this container in
time.sleep(wait_seconds) # cooldown for the wait plus its normal cooldown, and do the one-time HTTP
# re-check on its first regular check after the wait (main._check_one).
log.info(
"%-*s letting stack %s settle for %ds; other containers keep being checked",
width, name, key, wait_seconds,
)
state.record_restart(now, wait_seconds + container_config.cooldown_seconds)
if container_config.http_check.enabled: if container_config.http_check.enabled:
state.redeploy_recheck_at = now + timedelta(seconds=wait_seconds)
def run_post_redeploy_recheck(
*,
container_config: ContainerConfig,
state: ContainerState,
notifier: GotifyNotifier,
now: datetime,
) -> None:
"""One HTTP re-check after a stack redeploy has had time to settle.
Called by main._check_one once `state.redeploy_recheck_at` has passed;
alerts if the app still doesn't answer.
"""
state.redeploy_recheck_at = None
name = container_config.name
if not container_config.http_check.enabled:
return
recheck = check_http(container_config.http_check) recheck = check_http(container_config.http_check)
if recheck.ok: if recheck.ok:
log.info("recheck succeeded for %s", name) log.info("recheck succeeded for %s", name)
else: return
log.warning("recheck failed for %s: %s", name, recheck.message) log.warning("recheck failed for %s: %s", name, recheck.message)
notifier.notify( notifier.notify(
title=f"⚠️ Inspector: {name} recheck failed after redeploy", title=f"⚠️ Inspector: {name} recheck failed after redeploy",
message=( message=(
f"Portainer stack **{stack_name}** was redeployed, but the HTTP recheck " f"Portainer stack **{container_config.portainer_stack_name}** was redeployed, but the HTTP recheck "
f"for **{name}** still reports a failure.\n\n" f"for **{name}** still reports a failure.\n\n"
f"- Recheck result: *{recheck.message}*\n" f"- Recheck result: *{recheck.message}*\n"
f"- Time: {format_timestamp(now, with_zone=True)}" f"- Time: {format_timestamp(now, with_zone=True)}"
), ),
priority=6, priority=6,
) )
state.record_restart(now, container_config.cooldown_seconds)
+2
View File
@@ -29,6 +29,8 @@ class ContainerState:
restart_limit_notified: bool = False restart_limit_notified: bool = False
restarted_this_episode: bool = False restarted_this_episode: bool = False
last_check_at: datetime | None = None last_check_at: datetime | None = None
# After a stack redeploy: when to do the one-time HTTP re-check.
redeploy_recheck_at: datetime | None = None
def in_cooldown(self, now: datetime) -> bool: def in_cooldown(self, now: datetime) -> bool:
return self.cooldown_until is not None and now < self.cooldown_until return self.cooldown_until is not None and now < self.cooldown_until