diff --git a/inspector/inspector_qb.py b/inspector/inspector_qb.py new file mode 100755 index 0000000..7a148b7 --- /dev/null +++ b/inspector/inspector_qb.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 + +import time +import os +import datetime +import logging +import logging.handlers +import colorlog +import docker +import requests + +# Configuration +container_name = 'qbittorrent' + +interval_seconds = 60 +logs_folder = 'logs/qbittorrent' +cooldown_seconds = 300 +max_error_count = 5 + +# Create logs folder if it doesn't exist +if not os.path.exists(logs_folder): + os.makedirs(logs_folder) + +def get_log_file_path(): + return os.path.join(logs_folder, 'qbittorrent.log') + +def rotate_log_files(log_file): + now = datetime.datetime.now() + timestamp = now.strftime('%Y-%m-%d') + rotated_log_file = f'{timestamp}.log' + rotated_log_file = f'qbittorrent_{timestamp}.log' + rotated_log_path = os.path.join(logs_folder, rotated_log_file) + + # Rename the current log file to the rotated log file + os.rename(log_file, rotated_log_path) + + +def format_remaining_time(remaining_time): + minutes, seconds = divmod(remaining_time.seconds, 60) + return f'{minutes} minutes {seconds} seconds' + + +def check_container_health(container_name): + client = docker.from_env() + container = client.containers.get(container_name) + return container.attrs['State']['Health']['Status'].upper() + + +def restart_container(container_name): + client = docker.from_env() + container = client.containers.get(container_name) + container.restart() + +if __name__ == '__main__': + + last_log_date = None # Variable to keep track of the last log file creation date + + # Set the logging level to INFO + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + # Create a colorlog formatter + formatter = colorlog.ColoredFormatter( + '%(log_color)s%(asctime)s - %(message)s', + datefmt='%Y-%m-%d %I:%M:%S %p', + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'white,bg_red', + } + ) + + # Create a stream handler for console output + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + # Create a file handler for the current log file + log_file = get_log_file_path() + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # Create a timed rotating file handler for log rotation + rotating_file_handler = logging.handlers.TimedRotatingFileHandler( + log_file, + when='midnight', + interval=1, + backupCount=365, + atTime=datetime.time(0, 0) # Rotate at midnight + ) + rotating_file_handler.setFormatter(formatter) + logger.addHandler(rotating_file_handler) + + consecutive_error_count = 0 + reboot_triggered = False + last_reboot_time = None + cooldown_start_time = None + + + while True: + + + health_status = check_container_health(container_name) + + # Check if a new log file needs to be created after rotation + current_log_date = datetime.datetime.now().strftime('%Y-%m-%d') + if current_log_date != last_log_date: + # Perform log rotation if it's a new day + last_log_date = current_log_date + rotate_log_files(log_file) + + # Update the file handler with the new log file path + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(formatter) + + # Remove any existing file handlers and add the new file handler + for old_handler in logger.handlers[:]: + if isinstance(old_handler, logging.FileHandler): + logger.removeHandler(old_handler) + logger.addHandler(file_handler) + + + #logger.info('[INFO] - --------------------------------') + #logger.info('[INFO] - --------------------------------') + #logger.info(f'qbittorrent Health Status: {health_status}') + + + # Check if container Health Status is HEALTHY or not + if health_status != 'HEALTHY': + logger.warning(f'[WARN] - qbittorrent Health Status: {health_status}') + else: + logger.info(f'[INFO] - qbittorrent Health Status: {health_status}') + + + + # Check if container is not healthy and cooldown is not active + if health_status != 'HEALTHY' and not cooldown_start_time: + logger.warning('[WARN] - Container is not HEALTHY. Triggering error count.') + consecutive_error_count += 1 + + # Check if cooldown is not active and the container is not healthy during the current loop iteration + if not cooldown_start_time and (health_status != 'HEALTHY'): + logger.warning(f'[WARN] - Error Count: {consecutive_error_count}') + + # Check if consecutive error count reached the maximum + if consecutive_error_count >= max_error_count: + logger.critical(f'[CRIT] - Rebooting Container: {container_name} due to consecutive error count') + restart_container(container_name) + logger.warning('[WARN] - Container Restarted.') + reboot_triggered = True + consecutive_error_count = 0 + + + # Check if cooldown is active + if cooldown_start_time: + remaining_time = cooldown_start_time + datetime.timedelta(seconds=cooldown_seconds) - datetime.datetime.now() + remaining_time = max(remaining_time, datetime.timedelta()) + logger.warning(f'[WARN] - Cooldown Countdown: {format_remaining_time(remaining_time)} remaining.') + logger.warning(f'[WARN] - Error Count: {consecutive_error_count}') + + # Check if the cooldown period has passed + if datetime.datetime.now() > cooldown_start_time + datetime.timedelta(seconds=cooldown_seconds): + logger.error('[EROR] - Cooldown Period Ended') + logger.error('[EROR] - ---------------------') + cooldown_start_time = None + reboot_triggered = False # Set reboot_triggered to False + #logger.warning(f'cooldown_start_time should be set to NONE: {cooldown_start_time}') + consecutive_error_count = 0 # Reset error count + + # Check if reboot was triggered and cooldown is not active + if reboot_triggered and not cooldown_start_time: + #logger.warning(f'cooldown_start_time should be set to a time: ***{cooldown_start_time}***') + #logger.warning(f'reboot_triggered : ***{reboot_triggered}***') + logger.error('[EROR] - Cooldown Started.') + #set cooldown_start_time + cooldown_start_time = datetime.datetime.now() + + # Format cooldowntime and output it + remaining_time = cooldown_start_time + datetime.timedelta(seconds=cooldown_seconds) - datetime.datetime.now() + remaining_time = max(remaining_time, datetime.timedelta()) + logger.warning(f'[WARN] - Cooldown Countdown: {format_remaining_time(remaining_time)} remaining.') + + # Set error count to 0 + consecutive_error_count = 0 # Reset error count + + + # Check if status is HEALTHY + if health_status == 'HEALTHY': + # Reset error count + consecutive_error_count = 0 + + + logger.info('[INFO] - --------------------------------') + + time.sleep(interval_seconds) + + + + + + + + + diff --git a/ip_checker/ip_checker.py b/ip_checker/ip_checker.py new file mode 100755 index 0000000..6a69a86 --- /dev/null +++ b/ip_checker/ip_checker.py @@ -0,0 +1,273 @@ +#!/usr/bin/python3 +# +# +# +# +# + +import os +import time +import json +import logging +import datetime +import colorlog +import requests +import subprocess +import logging.handlers +from configparser import ConfigParser +from email.mime.text import MIMEText + +# Instantiate +config = ConfigParser() + +# Parse existing file +config.read('ip_checker_config.ini') + +# config file values +app_token = config.get('Pushover', 'APP_TOKEN') +user_key = config.get('Pushover', 'USER_KEY') +nfty_url = config.get('Nfty', 'NFTY_URL') +alert_method = config.get('Settings', 'ALERT_METHOD') + +# Docker container's name or ID +container_name = 'qbittorrent-openvpn' +dependent_container_name = 'qbittorrent' + +get_ip_address = 'https://ipinfo.io/ip' + +interval_seconds = 60 +logs_folder = 'logs' + +# Create logs folder if it doesn't exist +if not os.path.exists(logs_folder): + os.makedirs(logs_folder) + +def get_log_file_path(): + return os.path.join(logs_folder, 'ip_checker.log') + +def rotate_log_files(log_file): + now = datetime.datetime.now() + timestamp = now.strftime('%Y-%m-%d') + rotated_log_file = f'{timestamp}.log' + rotated_log_file = f'ip_checker_{timestamp}.log' + rotated_log_path = os.path.join(logs_folder, rotated_log_file) + + # Rename the current log file to the rotated log file + os.rename(log_file, rotated_log_path) + +def format_remaining_time(remaining_time): + minutes, seconds = divmod(remaining_time.seconds, 60) + return f'{minutes} minutes {seconds} seconds' + +def get_external_ip(): + try: + response = requests.get(f'{get_ip_address}') + response.raise_for_status() + return response.text.strip() + except requests.RequestException as e: + logger.warning(f'[WARN] - Error fetching external IP: {e}') + logger.info(' ') + return None + +def get_vpn_container_ip(container_name): + try: + cmd = ["docker", "exec", container_name, "curl", "-s", get_ip_address] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + logger.warning(f'[WARN] - Error fetching VPN container IP: {result.stderr}') + logger.info(' ') + return None + return result.stdout.strip() + except Exception as e: + logger.warning(f'[WARN] - Error: {e}') + logger.info(' ') + return None + +def stop_dependent_container(container_name): + """ + Stop the specified Docker container. + """ + try: + subprocess.run(["docker", "stop", container_name], check=True) + logger.info(f'[INFO] - Container {container_name} stopped.') + logger.info(' ') + except subprocess.CalledProcessError as e: + logger.warning(f'[WARN] - Failed stopping container {container_name}: {e}') + logger.info(' ') + +def send_ntfy_alert(message, urgent=False): + """ + Send a notification using ntfy. + """ + try: + topic = "autoheal" if urgent else "regular" + response = requests.post(f'{nfty_url}', data=message) + response.raise_for_status() + logger.info('[INFO] - nty notification sent.') + logger.info(' ') + except requests.RequestException as e: + logger.warning(f'[WARN] - Failed to send ntfy notification: {e}') + logger.info(' ') + +def send_pushover_alert(app_token, user_key): + + # Define notification title and message + title = "** IP CHECKER ALERT **" + message = f"URGENT!! VPN is down! IPs match" + + try: + url = "https://api.pushover.net/1/messages.json" + data = { + "token": app_token, + "user": user_key, + "title": title, + "html": "1", + "message": message, + } + response = requests.post(url, data=data) + if response.status_code == 200: + logger.info('[INFO] - Pushover notification sent.') + else: + logger.warning(f'[WARN] - Failed to send Pushover notification: {response.text}') + except Exception as e: + logger.warning(f'[WARN] - Failed to send Pushover notification: {e}') + + logger.info(' ') + + + +def send_alert(alert_method): + if alert_method == 'ntfy': + # Send urgent ntfy notification + send_ntfy_alert("Urgent: VPN is down! IPs match.", urgent=True) + + elif alert_method == 'pushover': + # Send urgent pushover notification + send_pushover_alert(app_token, user_key) + + # Stop the dependent container + stop_dependent_container(dependent_container_name) + + logger.critical(f'[CRIT] - Urgent: VPN is down! IPs match.') + logger.info(' ') + +def main(): + global last_log_date # Declare as global to modify the variable outside local scope + + # Variable to keep track of the last log file creation date + last_log_date = datetime.datetime.now().strftime('%Y-%m-%d') + + while True: + isp_ip = get_external_ip() + vpn_container_ip = get_vpn_container_ip(container_name) + + # Check if a new log file needs to be created after rotation + current_log_date = datetime.datetime.now().strftime('%Y-%m-%d') + if current_log_date != last_log_date: + # Perform log rotation if it's a new day + last_log_date = current_log_date + rotate_log_files(log_file) + + # Update the file handler with the new log file path + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(formatter) + + # Remove any existing file handlers and add the new file handler + for old_handler in logger.handlers[:]: + if isinstance(old_handler, logging.FileHandler): + logger.removeHandler(old_handler) + logger.addHandler(file_handler) + + if isp_ip and vpn_container_ip: + if isp_ip == vpn_container_ip: + send_alert(alert_method) + else: + # if both isp and vpn dont match just create a log entry that includes date, time, and IP + logger.info(f'[INFO] - VPN IP: {vpn_container_ip}') + logger.info(' ') + + # Wait for 2 minutes before next check + time.sleep(120) + +if __name__ == "__main__": + + + # Variable to keep track of the last log file creation date + last_log_date = None + + # Set the logging level to INFO + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + # Create a colorlog formatter + formatter = colorlog.ColoredFormatter( + '%(log_color)s%(asctime)s - %(message)s', + datefmt='%Y-%m-%d %I:%M:%S %p', + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'white,bg_red', + } + ) + + # Create a stream handler for console output + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + # Create a file handler for the current log file + log_file = get_log_file_path() + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + # Create a timed rotating file handler for log rotation + rotating_file_handler = logging.handlers.TimedRotatingFileHandler( + log_file, + when='midnight', + interval=1, + backupCount=365, + atTime=datetime.time(0, 0) # Rotate at midnight + ) + rotating_file_handler.setFormatter(formatter) + logger.addHandler(rotating_file_handler) + + + + main() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +