das/EO_Sevice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Load config, harden processing and notifications

Browse Source 
- Actually load configs/config.yaml at startup (init_config was commented out)
- Add timeouts and error handling to all HTTP calls; run blocking requests
  in threads; serialize processing with a lock; run initial check in background
- URL-encode Telegram messages and HTML-escape scraped data
- Fix GPV crash on empty schedule, 00:00 end time, date sorting and fallback format
- Parse PO table per row; fix daily reminder check and change detection
- Use per-instance timestamps in dataclasses; robust config value parsing
- Matrix: use v3 PUT send endpoint; strip secrets from logged errors
- Resolve templates/static/test paths relative to the project; use lifespan
- Move debug GPV data to test/data_gpv.json; add CLAUDE.md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Anry Das
2026-10-13 14:04:18 +03:00
co-authored by Claude Opus 5.5
parent e3367c6a2e
commit 22fc8b38bf
8 changed files with 683 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app_config.py
+96 -44
View File
@@ -7,13 +7,18 @@ import uvicorn
from config_file import read_yaml
APP_VERSION="1.0"
SCRIPT_PATH = os.path.dirname(__file__)
CONFIGS_DIR = SCRIPT_PATH + "/configs"
CONFIG_FILE_NAME = CONFIGS_DIR + "/config.yaml"
APP_VERSION = "1.0"
SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
CONFIGS_DIR = os.path.join(SCRIPT_PATH, "configs")
CONFIG_FILE_NAME = os.path.join(CONFIGS_DIR, "config.yaml")
TEMPLATES_DIR = os.path.join(SCRIPT_PATH, "templates")
STATIC_DIR = os.path.join(SCRIPT_PATH, "static")
TEST_DIR = os.path.join(SCRIPT_PATH, "test")
SECRETS_NAME = 'secrets.yaml'
CONFIG: Dict[str, Any] = {}
LOG_FORMAT = '%(asctime)s [%(levelname)s] - %(name)s: %(funcName)s[%(lineno)d] - %(message)s'
API_KEY = ''
IN_MEMORY_LOGS = False
IN_MEMORY_LOGS_LEN = 100
@@ -35,6 +40,8 @@ APARTMENT = ""
PROCESS_START_HOUR = 6
PROCESS_STOP_HOUR = 23
PROCESSING_MINUTES: List[int] = [5, 15, 30, 45, 55]
HTTP_TIMEOUT = 30 # seconds, for all outgoing requests
def read_app_config():
global CONFIG
@@ -46,8 +53,38 @@ def read_app_config():
CONFIG = read_yaml(CONFIG_FILE_NAME, SECRETS_NAME)
def get_config_value(cfg, section, key, default):
return cfg[section][key] if section in cfg else default
def get_section(cfg: Dict[str, Any], *path: str) -> Dict[str, Any]:
"""Returns nested section of config or empty dict if it is absent or not a mapping"""
section = cfg
for key in path:
section = section.get(key) if isinstance(section, dict) else None
if not isinstance(section, dict):
return {}
return section
def as_bool(value: Any) -> bool:
if isinstance(value, bool):
return value
return str(value).strip().lower() in ('true', 'yes', 'on', '1')
def as_int(value: Any, default: int, min_value: int = None, max_value: int = None) -> int:
try:
res = int(value)
except (TypeError, ValueError):
LOG.warning(f'Wrong integer value in config: {value!r}, default {default} is used')
return default
if (min_value is not None and res < min_value) or (max_value is not None and res > max_value):
LOG.warning(f'Value {res} is out of range [{min_value}, {max_value}], default {default} is used')
return default
return res
def as_str(value: Any) -> str:
return '' if value is None else str(value)
def parse_config():
global API_KEY, IN_MEMORY_LOGS, IN_MEMORY_LOGS_LEN, HOST, PORT, IS_DEBUG, \
@@ -55,42 +92,51 @@ def parse_config():
USE_MATRIX, MX_SERVER, MX_ROOM, MX_TOKEN, \
ACCOUNT, SETTLEMENT, STREET, HOUSE, BUILDING_PART_NUMBER, APARTMENT, \
PROCESS_START_HOUR, PROCESS_STOP_HOUR
HOST = CONFIG.get('application', {}).get('host', '')
if not HOST:
HOST = '0.0.0.0'
PORT = CONFIG.get('application', {}).get('port', 8000)
API_KEY = CONFIG.get('application').get('api-key', '')
IN_MEMORY_LOGS = str(CONFIG.get('application', {}).get('logs_ep', {}).get('enabled', False)).lower() == 'true'
IN_MEMORY_LOGS_LEN = CONFIG.get('application', {}).get('logs_ep', {}).get('max_records', 100)
IS_DEBUG = CONFIG.get('application').get('debug', False) == 'true'
TG_TOKEN = CONFIG.get('notifications', {}).get('tg', {}).get('token', '')
TG_CHAT = CONFIG.get('notifications', {}).get('tg', {}).get('chat_id', '')
USE_MATRIX = str(CONFIG.get('notifications', {}).get('matrix', {}).get('use_matrix', False)).lower() == 'true'
MX_SERVER = CONFIG.get('notifications', {}).get('matrix', {}).get('mx_server', '')
MX_ROOM = CONFIG.get('notifications', {}).get('matrix', {}).get('mx_room_id', '')
MX_TOKEN = CONFIG.get('notifications', {}).get('matrix', {}).get('mx_access_token', '')
ACCOUNT = CONFIG.get('data', {}).get('account', {})
SETTLEMENT = CONFIG.get('data', {}).get('settlement', {})
STREET = CONFIG.get('data', {}).get('street', {})
HOUSE = CONFIG.get('data', {}).get('house', {})
BUILDING_PART_NUMBER = CONFIG.get('data', {}).get('building_part_number', {})
APARTMENT = CONFIG.get('data', {}).get('apartment', {})
PROCESS_START_HOUR = CONFIG.get('application').get('process', {}).get('start_hour', 6)
PROCESS_STOP_HOUR = CONFIG.get('application').get('process', {}).get('stop_hour', 23)
if PROCESS_START_HOUR > PROCESS_STOP_HOUR:
PROCESS_STOP_HOUR = 23
app_section = get_section(CONFIG, 'application')
logs_section = get_section(app_section, 'logs_ep')
process_section = get_section(app_section, 'process')
tg_section = get_section(CONFIG, 'notifications', 'tg')
mx_section = get_section(CONFIG, 'notifications', 'matrix')
data_section = get_section(CONFIG, 'data')
HOST = as_str(app_section.get('host')) or '0.0.0.0'
PORT = as_int(app_section.get('port', 8000), 8000, 1, 65535)
API_KEY = as_str(app_section.get('api-key'))
IN_MEMORY_LOGS = as_bool(logs_section.get('enabled', False))
IN_MEMORY_LOGS_LEN = as_int(logs_section.get('max_records', 100), 100, 1)
IS_DEBUG = as_bool(app_section.get('debug', False))
TG_TOKEN = as_str(tg_section.get('token'))
TG_CHAT = as_str(tg_section.get('chat_id'))
USE_MATRIX = as_bool(mx_section.get('use_matrix', False))
MX_SERVER = as_str(mx_section.get('mx_server')).rstrip('/')
MX_ROOM = as_str(mx_section.get('mx_room_id'))
MX_TOKEN = as_str(mx_section.get('mx_access_token'))
ACCOUNT = as_str(data_section.get('account'))
SETTLEMENT = as_str(data_section.get('settlement'))
STREET = as_str(data_section.get('street'))
HOUSE = as_str(data_section.get('house'))
BUILDING_PART_NUMBER = as_str(data_section.get('building_part_number'))
APARTMENT = as_str(data_section.get('apartment'))
PROCESS_START_HOUR = as_int(process_section.get('start_hour', 6), 6, 0, 23)
PROCESS_STOP_HOUR = as_int(process_section.get('stop_hour', 23), 23, 1, 24)
if PROCESS_START_HOUR >= PROCESS_STOP_HOUR:
LOG.warning(f'start_hour {PROCESS_START_HOUR} >= stop_hour {PROCESS_STOP_HOUR}, stop_hour 24 is used')
PROCESS_STOP_HOUR = 24
if not TG_TOKEN or not TG_CHAT:
LOG.warning('Telegram token or chat_id is not configured')
if USE_MATRIX and not (MX_SERVER and MX_ROOM and MX_TOKEN):
LOG.warning('Matrix is enabled but not fully configured')
def get_uvicorn_config():
# global HOST, PORT
if HOST and PORT:
cfg = {
"host": f"{HOST}",
"port": PORT,
"access_log": True,
"log_level": "info",
"log_config": f"{CONFIGS_DIR}/log.yaml"
# "docs_url": "/api-docs"
"log_config": os.path.join(CONFIGS_DIR, "log.yaml")
}
else:
cfg = {
@@ -99,35 +145,41 @@ def get_uvicorn_config():
"reload": True,
"access_log": False,
"log_level": "info",
"log_config": f"{CONFIGS_DIR}/log.yaml"
# "docs_url": "/api-docs"
"log_config": os.path.join(CONFIGS_DIR, "log.yaml")
}
LOG.debug(f'got uvicorn_config: {cfg}')
return cfg
def log_config(): # Not used due to not working
def log_config(): # Not used due to not working
cfg = uvicorn.config.LOGGING_CONFIG
cfg["formatters"]["access"]["fmt"] = (
'%(asctime)s [%(levelname)s] - %(name)s: %(funcName)s[%(lineno)d] - %(message)s'
)
cfg["formatters"]["access"]["fmt"] = LOG_FORMAT
return cfg
LOG_LIST = []
LOG_LIST: List[str] = []
class ListHandler(logging.Handler):
def emit(self, record):
if IN_MEMORY_LOGS:
if not IN_MEMORY_LOGS:
return
try:
LOG_LIST.append(self.format(record))
while len(LOG_LIST) > IN_MEMORY_LOGS_LEN:
LOG_LIST.pop(0)
if len(LOG_LIST) > IN_MEMORY_LOGS_LEN:
del LOG_LIST[:len(LOG_LIST) - IN_MEMORY_LOGS_LEN]
except Exception:
self.handleError(record)
###
LOG = logging.getLogger(__name__)
handler = ListHandler()
handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] - %(name)s: %(funcName)s[%(lineno)d] - %(message)s'))
handler.setFormatter(logging.Formatter(LOG_FORMAT))
LOG.addHandler(handler)
def init_config():
read_app_config()
parse_config()