Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2a9c57b7d7 |
No files matched your search
@@ -7,6 +7,21 @@ Version numbers follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## [0.9.5] — 2026-10-28
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **A stack redeploy no longer freezes Inspector.** After a successful Portainer
|
||||||
|
redeploy, Inspector used to stop everything for `post_redeploy_wait_seconds`
|
||||||
|
(default 90 s): no other container was checked, dashboard times stood still, and
|
||||||
|
stopping Inspector during the wait wasn't clean. Now the redeployed app is left to
|
||||||
|
settle while every other container keeps being checked, and its one-time HTTP
|
||||||
|
re-check (with the "recheck failed" alert) happens on its first check after the wait.
|
||||||
|
- The redeployed app's cooldown now starts after the settle time. Before, the wait
|
||||||
|
ate into it (with the defaults, 300 s cooldown became 210 s).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## [0.9.4] — 2026-10-28
|
## [0.9.4] — 2026-10-28
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
__version__ = "0.9.4"
|
__version__ = "0.9.5"
|
||||||
@@ -356,8 +356,9 @@ folder (or a backup of it) can read the token.</p>
|
|||||||
<tr><td>stack_redeploy_cooldown_seconds</td><td>Minimum time between two redeploys of the same
|
<tr><td>stack_redeploy_cooldown_seconds</td><td>Minimum time between two redeploys of the same
|
||||||
stack (default 300). Several apps often share a stack; this stops them triggering a redeploy
|
stack (default 300). Several apps often share a stack; this stops them triggering a redeploy
|
||||||
each.</td></tr>
|
each.</td></tr>
|
||||||
<tr><td>post_redeploy_wait_seconds</td><td>How long Inspector waits after a redeploy for the
|
<tr><td>post_redeploy_wait_seconds</td><td>How long Inspector gives the stack to come up after
|
||||||
stack to come up before judging it again (default 90).</td></tr>
|
a redeploy before judging that app again (default 90). Everything else keeps being checked
|
||||||
|
meanwhile.</td></tr>
|
||||||
</table>
|
</table>
|
||||||
|
|
||||||
<h3>If Test connection fails</h3>
|
<h3>If Test connection fails</h3>
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ from .maintenance import MaintenanceTracker
|
|||||||
from .notification_mute import NotificationMuteTracker
|
from .notification_mute import NotificationMuteTracker
|
||||||
from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier
|
from .notifier import GotifyNotifier, MultiNotifier, NullNotifier, PushoverNotifier
|
||||||
from .portainer_client import PortainerClient
|
from .portainer_client import PortainerClient
|
||||||
from .recovery import perform_recovery
|
from .recovery import perform_recovery, run_post_redeploy_recheck
|
||||||
from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient
|
from .uptime_kuma import HeartbeatSnapshot, UptimeKumaClient
|
||||||
from .state import (
|
from .state import (
|
||||||
ContainerState,
|
ContainerState,
|
||||||
@@ -153,6 +153,11 @@ def _check_one(
|
|||||||
if notification_mute.is_muted(name):
|
if notification_mute.is_muted(name):
|
||||||
notifier = NullNotifier()
|
notifier = NullNotifier()
|
||||||
|
|
||||||
|
if state.redeploy_recheck_at is not None and now >= state.redeploy_recheck_at:
|
||||||
|
run_post_redeploy_recheck(
|
||||||
|
container_config=container_config, state=state, notifier=notifier, now=now,
|
||||||
|
)
|
||||||
|
|
||||||
if state.previous_status != state.current_status:
|
if state.previous_status != state.current_status:
|
||||||
log.info(
|
log.info(
|
||||||
"%-*s status: %s -> %s%s",
|
"%-*s status: %s -> %s%s",
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
from .config import AppConfig, ContainerConfig, PortainerConfig
|
from .config import AppConfig, ContainerConfig, PortainerConfig
|
||||||
@@ -522,24 +521,46 @@ def _perform_portainer_redeploy(
|
|||||||
priority=4,
|
priority=4,
|
||||||
)
|
)
|
||||||
|
|
||||||
log.info("waiting %d seconds after stack redeploy", wait_seconds)
|
# Don't block the loop while the stack settles: keep this container in
|
||||||
time.sleep(wait_seconds)
|
# cooldown for the wait plus its normal cooldown, and do the one-time HTTP
|
||||||
|
# re-check on its first regular check after the wait (main._check_one).
|
||||||
|
log.info(
|
||||||
|
"%-*s letting stack %s settle for %ds; other containers keep being checked",
|
||||||
|
width, name, key, wait_seconds,
|
||||||
|
)
|
||||||
|
state.record_restart(now, wait_seconds + container_config.cooldown_seconds)
|
||||||
if container_config.http_check.enabled:
|
if container_config.http_check.enabled:
|
||||||
|
state.redeploy_recheck_at = now + timedelta(seconds=wait_seconds)
|
||||||
|
|
||||||
|
|
||||||
|
def run_post_redeploy_recheck(
|
||||||
|
*,
|
||||||
|
container_config: ContainerConfig,
|
||||||
|
state: ContainerState,
|
||||||
|
notifier: GotifyNotifier,
|
||||||
|
now: datetime,
|
||||||
|
) -> None:
|
||||||
|
"""One HTTP re-check after a stack redeploy has had time to settle.
|
||||||
|
|
||||||
|
Called by main._check_one once `state.redeploy_recheck_at` has passed;
|
||||||
|
alerts if the app still doesn't answer.
|
||||||
|
"""
|
||||||
|
state.redeploy_recheck_at = None
|
||||||
|
name = container_config.name
|
||||||
|
if not container_config.http_check.enabled:
|
||||||
|
return
|
||||||
recheck = check_http(container_config.http_check)
|
recheck = check_http(container_config.http_check)
|
||||||
if recheck.ok:
|
if recheck.ok:
|
||||||
log.info("recheck succeeded for %s", name)
|
log.info("recheck succeeded for %s", name)
|
||||||
else:
|
return
|
||||||
log.warning("recheck failed for %s: %s", name, recheck.message)
|
log.warning("recheck failed for %s: %s", name, recheck.message)
|
||||||
notifier.notify(
|
notifier.notify(
|
||||||
title=f"⚠️ Inspector: {name} recheck failed after redeploy",
|
title=f"⚠️ Inspector: {name} recheck failed after redeploy",
|
||||||
message=(
|
message=(
|
||||||
f"Portainer stack **{stack_name}** was redeployed, but the HTTP recheck "
|
f"Portainer stack **{container_config.portainer_stack_name}** was redeployed, but the HTTP recheck "
|
||||||
f"for **{name}** still reports a failure.\n\n"
|
f"for **{name}** still reports a failure.\n\n"
|
||||||
f"- Recheck result: *{recheck.message}*\n"
|
f"- Recheck result: *{recheck.message}*\n"
|
||||||
f"- Time: {format_timestamp(now, with_zone=True)}"
|
f"- Time: {format_timestamp(now, with_zone=True)}"
|
||||||
),
|
),
|
||||||
priority=6,
|
priority=6,
|
||||||
)
|
)
|
||||||
|
|
||||||
state.record_restart(now, container_config.cooldown_seconds)
|
|
||||||
@@ -29,6 +29,8 @@ class ContainerState:
|
|||||||
restart_limit_notified: bool = False
|
restart_limit_notified: bool = False
|
||||||
restarted_this_episode: bool = False
|
restarted_this_episode: bool = False
|
||||||
last_check_at: datetime | None = None
|
last_check_at: datetime | None = None
|
||||||
|
# After a stack redeploy: when to do the one-time HTTP re-check.
|
||||||
|
redeploy_recheck_at: datetime | None = None
|
||||||
|
|
||||||
def in_cooldown(self, now: datetime) -> bool:
|
def in_cooldown(self, now: datetime) -> bool:
|
||||||
return self.cooldown_until is not None and now < self.cooldown_until
|
return self.cooldown_until is not None and now < self.cooldown_until
|
||||||
|
|||||||
Reference in new issue
Block a user