Inspector 0.9.5: stack redeploy no longer blocks other checks
This commit is contained in:
1 parent
6aed0cf9a7
commit
2a9c57b7d7
6 files changed
+68
-24
No files matched your search
@@ -7,6 +7,21 @@ Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html
|
||||
|
||||
---
|
||||
|
||||
## [0.9.5] — 2026-10-28
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A stack redeploy no longer freezes Inspector.** After a successful Portainer
|
||||
redeploy, Inspector used to stop everything for `post_redeploy_wait_seconds`
|
||||
(default 90 s): no other container was checked, dashboard times stood still, and
|
||||
stopping Inspector during the wait wasn't clean. Now the redeployed app is left to
|
||||
settle while every other container keeps being checked, and its one-time HTTP
|
||||
re-check (with the "recheck failed" alert) happens on its first check after the wait.
|
||||
- The redeployed app's cooldown now starts after the settle time. Before, the wait
|
||||
ate into it (with the defaults, 300 s cooldown became 210 s).
|
||||
|
||||
---
|
||||
|
||||
## [0.9.4] — 2026-10-28
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -1 +1 @@
|
||||
__version__ = "0.9.4"
|
||||
__version__ = "0.9.5"
|
||||
@@ -356,8 +356,9 @@ folder (or a backup of it) can read the token.</p>
|
||||
<tr><td>stack_redeploy_cooldown_seconds</td><td>Minimum time between two redeploys of the same
|
||||
stack (default 300). Several apps often share a stack; this stops them triggering a redeploy
|
||||
each.</td></tr>
|
||||
<tr><td>post_redeploy_wait_seconds</td><td>How long Inspector waits after a redeploy for the
|
||||
stack to come up before judging it again (default 90).</td></tr>
|
||||
<tr><td>post_redeploy_wait_seconds</td><td>How long Inspector gives the stack to come up after
|
||||
a redeploy before judging that app again (default 90). Everything else keeps being checked
|
||||
meanwhile.</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>If Test connection fails</h3>
|
||||
|
||||
@@ -41,7 +41,7 @@ from .maintenance import MaintenanceTracker
|
||||
from .notification_mute import NotificationMuteTracker
|
||||
from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier
|
||||
from .portainer_client import PortainerClient
|
||||
from .recovery import perform_recovery
|
||||
from .recovery import perform_recovery, run_post_redeploy_recheck
|
||||
from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient
|
||||
from .state import (
|
||||
ContainerState,
|
||||
@@ -153,6 +153,11 @@ def _check_one(
|
||||
if notification_mute.is_muted(name):
|
||||
notifier = NullNotifier()
|
||||
|
||||
if state.redeploy_recheck_at is not None and now >= state.redeploy_recheck_at:
|
||||
run_post_redeploy_recheck(
|
||||
container_config=container_config, state=state, notifier=notifier, now=now,
|
||||
)
|
||||
|
||||
if state.previous_status != state.current_status:
|
||||
log.info(
|
||||
"%-*s status: %s -> %s%s",
|
||||
|
||||
+41
-20
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from .config import AppConfig, ContainerConfig, PortainerConfig
|
||||
@@ -522,24 +521,46 @@ def _perform_portainer_redeploy(
|
||||
priority=4,
|
||||
)
|
||||
|
||||
log.info("waiting %d seconds after stack redeploy", wait_seconds)
|
||||
time.sleep(wait_seconds)
|
||||
|
||||
# Don't block the loop while the stack settles: keep this container in
|
||||
# cooldown for the wait plus its normal cooldown, and do the one-time HTTP
|
||||
# re-check on its first regular check after the wait (main._check_one).
|
||||
log.info(
|
||||
"%-*s letting stack %s settle for %ds; other containers keep being checked",
|
||||
width, name, key, wait_seconds,
|
||||
)
|
||||
state.record_restart(now, wait_seconds + container_config.cooldown_seconds)
|
||||
if container_config.http_check.enabled:
|
||||
recheck = check_http(container_config.http_check)
|
||||
if recheck.ok:
|
||||
log.info("recheck succeeded for %s", name)
|
||||
else:
|
||||
log.warning("recheck failed for %s: %s", name, recheck.message)
|
||||
notifier.notify(
|
||||
title=f"⚠️ Inspector: {name} recheck failed after redeploy",
|
||||
message=(
|
||||
f"Portainer stack **{stack_name}** was redeployed, but the HTTP recheck "
|
||||
f"for **{name}** still reports a failure.\n\n"
|
||||
f"- Recheck result: *{recheck.message}*\n"
|
||||
f"- Time: {format_timestamp(now, with_zone=True)}"
|
||||
),
|
||||
priority=6,
|
||||
)
|
||||
state.redeploy_recheck_at = now + timedelta(seconds=wait_seconds)
|
||||
|
||||
state.record_restart(now, container_config.cooldown_seconds)
|
||||
|
||||
def run_post_redeploy_recheck(
|
||||
*,
|
||||
container_config: ContainerConfig,
|
||||
state: ContainerState,
|
||||
notifier: GotifyNotifier,
|
||||
now: datetime,
|
||||
) -> None:
|
||||
"""One HTTP re-check after a stack redeploy has had time to settle.
|
||||
|
||||
Called by main._check_one once `state.redeploy_recheck_at` has passed;
|
||||
alerts if the app still doesn't answer.
|
||||
"""
|
||||
state.redeploy_recheck_at = None
|
||||
name = container_config.name
|
||||
if not container_config.http_check.enabled:
|
||||
return
|
||||
recheck = check_http(container_config.http_check)
|
||||
if recheck.ok:
|
||||
log.info("recheck succeeded for %s", name)
|
||||
return
|
||||
log.warning("recheck failed for %s: %s", name, recheck.message)
|
||||
notifier.notify(
|
||||
title=f"⚠️ Inspector: {name} recheck failed after redeploy",
|
||||
message=(
|
||||
f"Portainer stack **{container_config.portainer_stack_name}** was redeployed, but the HTTP recheck "
|
||||
f"for **{name}** still reports a failure.\n\n"
|
||||
f"- Recheck result: *{recheck.message}*\n"
|
||||
f"- Time: {format_timestamp(now, with_zone=True)}"
|
||||
),
|
||||
priority=6,
|
||||
)
|
||||
@@ -29,6 +29,8 @@ class ContainerState:
|
||||
restart_limit_notified: bool = False
|
||||
restarted_this_episode: bool = False
|
||||
last_check_at: datetime | None = None
|
||||
# After a stack redeploy: when to do the one-time HTTP re-check.
|
||||
redeploy_recheck_at: datetime | None = None
|
||||
|
||||
def in_cooldown(self, now: datetime) -> bool:
|
||||
return self.cooldown_until is not None and now < self.cooldown_until
|
||||
|
||||
Reference in new issue
Block a user