Inspector / ip_checker
> adding scripts to repo
This commit is contained in:
210
inspector/inspector_qb.py
Executable file
210
inspector/inspector_qb.py
Executable file
@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import time
|
||||
import os
|
||||
import datetime
|
||||
import logging
|
||||
import logging.handlers
|
||||
import colorlog
|
||||
import docker
|
||||
import requests
|
||||
|
||||
# Configuration
|
||||
container_name = 'qbittorrent'
|
||||
|
||||
interval_seconds = 60
|
||||
logs_folder = 'logs/qbittorrent'
|
||||
cooldown_seconds = 300
|
||||
max_error_count = 5
|
||||
|
||||
# Create logs folder if it doesn't exist
|
||||
if not os.path.exists(logs_folder):
|
||||
os.makedirs(logs_folder)
|
||||
|
||||
def get_log_file_path():
|
||||
return os.path.join(logs_folder, 'qbittorrent.log')
|
||||
|
||||
def rotate_log_files(log_file):
|
||||
now = datetime.datetime.now()
|
||||
timestamp = now.strftime('%Y-%m-%d')
|
||||
rotated_log_file = f'{timestamp}.log'
|
||||
rotated_log_file = f'qbittorrent_{timestamp}.log'
|
||||
rotated_log_path = os.path.join(logs_folder, rotated_log_file)
|
||||
|
||||
# Rename the current log file to the rotated log file
|
||||
os.rename(log_file, rotated_log_path)
|
||||
|
||||
|
||||
def format_remaining_time(remaining_time):
|
||||
minutes, seconds = divmod(remaining_time.seconds, 60)
|
||||
return f'{minutes} minutes {seconds} seconds'
|
||||
|
||||
|
||||
def check_container_health(container_name):
|
||||
client = docker.from_env()
|
||||
container = client.containers.get(container_name)
|
||||
return container.attrs['State']['Health']['Status'].upper()
|
||||
|
||||
|
||||
def restart_container(container_name):
|
||||
client = docker.from_env()
|
||||
container = client.containers.get(container_name)
|
||||
container.restart()
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
last_log_date = None # Variable to keep track of the last log file creation date
|
||||
|
||||
# Set the logging level to INFO
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
# Create a colorlog formatter
|
||||
formatter = colorlog.ColoredFormatter(
|
||||
'%(log_color)s%(asctime)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %I:%M:%S %p',
|
||||
log_colors={
|
||||
'DEBUG': 'cyan',
|
||||
'INFO': 'green',
|
||||
'WARNING': 'yellow',
|
||||
'ERROR': 'red',
|
||||
'CRITICAL': 'white,bg_red',
|
||||
}
|
||||
)
|
||||
|
||||
# Create a stream handler for console output
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(logging.INFO)
|
||||
console_handler.setFormatter(formatter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# Create a file handler for the current log file
|
||||
log_file = get_log_file_path()
|
||||
file_handler = logging.FileHandler(log_file)
|
||||
file_handler.setLevel(logging.INFO)
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
# Create a timed rotating file handler for log rotation
|
||||
rotating_file_handler = logging.handlers.TimedRotatingFileHandler(
|
||||
log_file,
|
||||
when='midnight',
|
||||
interval=1,
|
||||
backupCount=365,
|
||||
atTime=datetime.time(0, 0) # Rotate at midnight
|
||||
)
|
||||
rotating_file_handler.setFormatter(formatter)
|
||||
logger.addHandler(rotating_file_handler)
|
||||
|
||||
consecutive_error_count = 0
|
||||
reboot_triggered = False
|
||||
last_reboot_time = None
|
||||
cooldown_start_time = None
|
||||
|
||||
|
||||
while True:
|
||||
|
||||
|
||||
health_status = check_container_health(container_name)
|
||||
|
||||
# Check if a new log file needs to be created after rotation
|
||||
current_log_date = datetime.datetime.now().strftime('%Y-%m-%d')
|
||||
if current_log_date != last_log_date:
|
||||
# Perform log rotation if it's a new day
|
||||
last_log_date = current_log_date
|
||||
rotate_log_files(log_file)
|
||||
|
||||
# Update the file handler with the new log file path
|
||||
file_handler = logging.FileHandler(log_file)
|
||||
file_handler.setLevel(logging.INFO)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
# Remove any existing file handlers and add the new file handler
|
||||
for old_handler in logger.handlers[:]:
|
||||
if isinstance(old_handler, logging.FileHandler):
|
||||
logger.removeHandler(old_handler)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
|
||||
#logger.info('[INFO] - --------------------------------')
|
||||
#logger.info('[INFO] - --------------------------------')
|
||||
#logger.info(f'qbittorrent Health Status: {health_status}')
|
||||
|
||||
|
||||
# Check if container Health Status is HEALTHY or not
|
||||
if health_status != 'HEALTHY':
|
||||
logger.warning(f'[WARN] - qbittorrent Health Status: {health_status}')
|
||||
else:
|
||||
logger.info(f'[INFO] - qbittorrent Health Status: {health_status}')
|
||||
|
||||
|
||||
|
||||
# Check if container is not healthy and cooldown is not active
|
||||
if health_status != 'HEALTHY' and not cooldown_start_time:
|
||||
logger.warning('[WARN] - Container is not HEALTHY. Triggering error count.')
|
||||
consecutive_error_count += 1
|
||||
|
||||
# Check if cooldown is not active and the container is not healthy during the current loop iteration
|
||||
if not cooldown_start_time and (health_status != 'HEALTHY'):
|
||||
logger.warning(f'[WARN] - Error Count: {consecutive_error_count}')
|
||||
|
||||
# Check if consecutive error count reached the maximum
|
||||
if consecutive_error_count >= max_error_count:
|
||||
logger.critical(f'[CRIT] - Rebooting Container: {container_name} due to consecutive error count')
|
||||
restart_container(container_name)
|
||||
logger.warning('[WARN] - Container Restarted.')
|
||||
reboot_triggered = True
|
||||
consecutive_error_count = 0
|
||||
|
||||
|
||||
# Check if cooldown is active
|
||||
if cooldown_start_time:
|
||||
remaining_time = cooldown_start_time + datetime.timedelta(seconds=cooldown_seconds) - datetime.datetime.now()
|
||||
remaining_time = max(remaining_time, datetime.timedelta())
|
||||
logger.warning(f'[WARN] - Cooldown Countdown: {format_remaining_time(remaining_time)} remaining.')
|
||||
logger.warning(f'[WARN] - Error Count: {consecutive_error_count}')
|
||||
|
||||
# Check if the cooldown period has passed
|
||||
if datetime.datetime.now() > cooldown_start_time + datetime.timedelta(seconds=cooldown_seconds):
|
||||
logger.error('[EROR] - Cooldown Period Ended')
|
||||
logger.error('[EROR] - ---------------------')
|
||||
cooldown_start_time = None
|
||||
reboot_triggered = False # Set reboot_triggered to False
|
||||
#logger.warning(f'cooldown_start_time should be set to NONE: {cooldown_start_time}')
|
||||
consecutive_error_count = 0 # Reset error count
|
||||
|
||||
# Check if reboot was triggered and cooldown is not active
|
||||
if reboot_triggered and not cooldown_start_time:
|
||||
#logger.warning(f'cooldown_start_time should be set to a time: ***{cooldown_start_time}***')
|
||||
#logger.warning(f'reboot_triggered : ***{reboot_triggered}***')
|
||||
logger.error('[EROR] - Cooldown Started.')
|
||||
#set cooldown_start_time
|
||||
cooldown_start_time = datetime.datetime.now()
|
||||
|
||||
# Format cooldowntime and output it
|
||||
remaining_time = cooldown_start_time + datetime.timedelta(seconds=cooldown_seconds) - datetime.datetime.now()
|
||||
remaining_time = max(remaining_time, datetime.timedelta())
|
||||
logger.warning(f'[WARN] - Cooldown Countdown: {format_remaining_time(remaining_time)} remaining.')
|
||||
|
||||
# Set error count to 0
|
||||
consecutive_error_count = 0 # Reset error count
|
||||
|
||||
|
||||
# Check if status is HEALTHY
|
||||
if health_status == 'HEALTHY':
|
||||
# Reset error count
|
||||
consecutive_error_count = 0
|
||||
|
||||
|
||||
logger.info('[INFO] - --------------------------------')
|
||||
|
||||
time.sleep(interval_seconds)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user