import asyncio import html import json import os import re import uuid from datetime import datetime, timedelta from typing import List, Optional from urllib.parse import quote import requests import urllib3 from lxml import etree import app_config as cfg from model import POData, GPVData, GPV_DATE_FORMAT STATUS_SEND_OK = 0 STATUS_ERROR_SEND_TO_TG = 20 STATUS_ERROR_GPV_RESPONSE = 10 STATUS_ERROR_PO_RESPONSE = 30 STATUS_EMPTY_CONFIG = 5 STATUS_ERROR_SEND_TO_MX = 40 PARSED_PO_DATAS: List[POData] = [] PARSED_GPV_DATAS: List[GPVData] = [] LOG = cfg.LOG # be-svitlo.oe.if.ua is requested without certificate verification urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) DIGIT_CHARS = [f'{d}\U000020E3' for d in range(10)] def safe_error(e: Exception) -> str: """Error description without secrets (request URLs may contain the TG token)""" msg = f'{type(e).__name__}: {e}' for secret in (cfg.TG_TOKEN, cfg.MX_TOKEN): if secret: msg = msg.replace(secret, '***') return msg def send_message_to_tg_get(msg: str) -> int: """Returns HTTP status code of Telegram response or 0 if the request failed""" if cfg.IS_DEBUG: print(f"Send to TG GET: {msg}") return 200 if not cfg.TG_TOKEN or not cfg.TG_CHAT: LOG.error('Telegram is not configured') return 0 try: response = requests.get( f"https://api.telegram.org/bot{cfg.TG_TOKEN}/sendMessage", params={'chat_id': cfg.TG_CHAT, 'parse_mode': 'html', 'text': msg}, timeout=cfg.HTTP_TIMEOUT) except requests.RequestException as e: LOG.error(f'Error sending TG message: {safe_error(e)}') return 0 if response.status_code != 200: LOG.error(f'TG response code={response.status_code}: {response.text[:500]}') return response.status_code def send_message_to_tg_post(msg: str) -> int: """Returns HTTP status code of Telegram response or 0 if the request failed""" if cfg.IS_DEBUG: print(f"Send to TG POST: {msg}") return 200 if not cfg.TG_TOKEN or not cfg.TG_CHAT: LOG.error('Telegram is not configured') return 0 try: return requests.post( url=f'https://api.telegram.org/bot{cfg.TG_TOKEN}/sendMessage', data={'chat_id': cfg.TG_CHAT, 'text': msg, 'parse_mode': 'html'}, timeout=cfg.HTTP_TIMEOUT ).status_code except requests.RequestException as e: LOG.error(f'Error sending TG message: {safe_error(e)}') return 0 def send_matrix_notification(message: str) -> bool: """Send notification to Matrix""" if not cfg.USE_MATRIX: return False if cfg.IS_DEBUG: print(f"Send to MX: {message}") return True if not (cfg.MX_SERVER and cfg.MX_ROOM and cfg.MX_TOKEN): LOG.error('Matrix is not configured') return False url = (f"{cfg.MX_SERVER}/_matrix/client/v3/rooms/{quote(cfg.MX_ROOM, safe='')}" f"/send/m.room.message/{uuid.uuid4().hex}") headers = { 'Authorization': f'Bearer {cfg.MX_TOKEN}', 'Content-Type': 'application/json' } payload = { 'msgtype': 'm.text', 'body': html.unescape(re.sub(r'<[^>]+>', '', message)), 'format': 'org.matrix.custom.html', 'formatted_body': message.replace('\n', '
') } try: response = requests.put(url, headers=headers, json=payload, timeout=cfg.HTTP_TIMEOUT) except requests.RequestException as e: LOG.error(f"Error sending Matrix notification: {safe_error(e)}") return False if response.status_code != 200: LOG.error(f'MX response code={response.status_code}: {response.text[:500]}') return response.status_code == 200 def notify(message: str) -> bool: """Sends message to all configured messengers. Returns True if it was delivered at least to one of them""" delivered = False if send_message_to_tg_get(message) == 200: delivered = True LOG.debug(f'TG message send {message}') else: LOG.error(f'TG send error code: {STATUS_ERROR_SEND_TO_TG}') if cfg.USE_MATRIX: if send_matrix_notification(message): delivered = True else: LOG.error(f'MX send error code: {STATUS_ERROR_SEND_TO_MX}') return delivered def get_GPV_response_from_oe(account): url = 'https://be-svitlo.oe.if.ua/schedule-by-search' headers = { "Host": "be-svitlo.oe.if.ua", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0", "Accept": "application/json, text/plain, */*", "Accept-Language": "en-US,en;q=0.5", "Content-Type": "application/x-www-form-urlencoded;charset=utf-8", "Referer": "https://svitlo.oe.if.ua/", "Origin": "https://svitlo.oe.if.ua", "Sec-Fetch-Dest": "empty", "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-site", "Connection": "keep-alive", "Priority": "u=0", } data = { "accountNumber": account, "userSearchChoice": "pob", "address": "" } return requests.post(url, headers=headers, data=data, verify=False, timeout=cfg.HTTP_TIMEOUT) def get_PO_response(): # Power Outage utf8 = '✓' url = 'https://oe.if.ua/uk/shutdowns_table' params = { 'utf8': utf8, 'settlement': cfg.SETTLEMENT, 'street': cfg.STREET, 'house_number': cfg.HOUSE, 'building_part_number': cfg.BUILDING_PART_NUMBER, 'apartment': cfg.APARTMENT, 'commit': 'Пошук' } headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36' } return requests.get(url=url, headers=headers, params=params, timeout=cfg.HTTP_TIMEOUT) def has_numbers(in_str): return any(char.isdigit() for char in str(in_str)) def get_utf_chars(number, is_time=False): """Replaces ASCII digits with keycap emoji digits. is_time is kept for compatibility""" return ''.join(DIGIT_CHARS[int(c)] if c in '0123456789' else c for c in str(number)) def get_digit_message(arr, is_time=False): first = arr[0] if len(arr) > 0 else 0 last = arr[1] if len(arr) > 1 else 0 return f'з {get_utf_chars(first, is_time)} до {get_utf_chars(last, is_time)}' def get_GPV_message(approved_from, event_date, hours_off, hours_on, outage_queue): queue = get_utf_chars(html.escape(str(outage_queue))) message = f'\U000026A1 ГПВ на {html.escape(str(event_date))} для {queue} черги' if len(hours_off) > 0: message += '\nВимкнення:' for hour_off, hour_on in zip(hours_off, hours_on): message += '\n' + get_digit_message([html.escape(str(hour_off)), html.escape(str(hour_on))], True) else: message += '\nВимкнень немає \U0001F44C \U0001F483' message += f'\n{html.escape(str(approved_from))}' return message def get_address(city, street, house): return f'{city}, {street}, {house}' def is_outage_date_before_or_same(stop_date): """True if the outage is not finished before today (or the date can't be parsed)""" try: parsed_date = datetime.strptime(stop_date, "%Y.%m.%d").date() except (TypeError, ValueError): LOG.warning(f'Wrong stop date: {stop_date!r}') return True return parsed_date >= datetime.now().date() def get_row_text(row, path: str) -> str: return ' '.join(t.strip() for t in row.xpath(path) if t and t.strip()) def parse_po_data(root) -> List[POData]: res: List[POData] = [] for row in root.xpath("//div[@class='table-row flex']"): po_data = POData( city=get_row_text(row, "./div[@class='city']//text()"), street=get_row_text(row, "./div[@class='street']//text()"), house=get_row_text(row, "./div[@class='house_number']//text()"), po_type=get_row_text(row, "./div[@class='type-shutdown']//text()"), reason=get_row_text(row, "./div[@class='reason-shutdown reason-text-container']/div[@class='reason-text-hide']//text()"), placement_date=get_row_text(row, "./div[@class='placement_date']/div[@class='date']//text()"), start_date=get_row_text(row, "./div[@class='shutdown_date']/div[@class='date']//text()"), stop_date=get_row_text(row, "./div[@class='turn_on_date']/div[@class='date']//text()"), start_time=get_row_text(row, "./div[@class='shutdown_date']/div[@class='time']//text()"), stop_time=get_row_text(row, "./div[@class='turn_on_date']/div[@class='time']//text()")) if not po_data.start_ts: LOG.warning(f'PO row skipped due to wrong start date/time: {po_data}') continue if not is_outage_date_before_or_same(po_data.stop_date): continue if po_data not in res: res.append(po_data) res.sort(key=lambda pd: pd.start_ts) return res def load_last_datas(file_name): if os.path.isfile(file_name): with open(file_name, 'r', encoding='utf-8') as f: return f.read() else: return '' def get_parsed_po_data(): return PARSED_PO_DATAS def get_parsed_gpv_data(): return PARSED_GPV_DATAS def gpv_date_key(entry: dict): try: return datetime.strptime(str(entry.get('eventDate')), GPV_DATE_FORMAT) except ValueError: return datetime.max def parse_gpv_data(x) -> List[GPVData]: if not isinstance(x, dict): raise ValueError(f'Unexpected GPV response type: {type(x).__name__}') current = x.get('current') or {} if current.get('queue') is None: LOG.warning(f'No outage queue in GPV response: {current}') return [] outage_queue = str(current['queue']) if current.get('subQueue'): outage_queue += '.' + str(current['subQueue']) schedule = x.get('schedule') if not isinstance(schedule, list): return [] res: List[GPVData] = [] for entry in sorted((e for e in schedule if isinstance(e, dict)), key=gpv_date_key): queues = entry.get('queues') event_date = entry.get('eventDate') if not isinstance(queues, dict) or not event_date: continue if outage_queue not in queues: LOG.warning(f'No queue {outage_queue} in GPV schedule for {event_date}') continue approved_since = entry.get('scheduleApprovedSince') approved_from = f'Запроваджено {approved_since}' if approved_since else 'Hемає даних про дату запровадження' hours_list = queues.get(outage_queue) or [] if cfg.IS_DEBUG: print(f'hours_list{hours_list}, approved_from={approved_from}, event_date={event_date}') hours_off = [] hours_on = [] for h in hours_list: if isinstance(h, dict) and h.get("status") == 1 and h.get("from") and h.get("to"): hours_off.append(h["from"]) hours_on.append(h["to"]) if cfg.IS_DEBUG: print(f'hours_off = {hours_off}') print(f'hours_on = {hours_on}') gpv_data = GPVData(approved_from=approved_from, event_date=event_date, hours_off=hours_off, hours_on=hours_on, outage_queue=outage_queue) if gpv_data.actual_till_ts is None: LOG.warning(f'GPV entry skipped due to wrong event date: {gpv_data}') continue res.append(gpv_data) return res def fetch_gpv_json(): if cfg.IS_DEBUG: with open(os.path.join(cfg.TEST_DIR, 'data_gpv.json'), 'r', encoding='utf-8') as f: return json.load(f) if not cfg.ACCOUNT: LOG.error('Account number is not configured') return None response = get_GPV_response_from_oe(cfg.ACCOUNT) if response.status_code != 200: LOG.error(f'GPV response code={response.status_code}') return None return response.json() async def process_gpv(): try: x = await asyncio.to_thread(fetch_gpv_json) if x is None: return STATUS_ERROR_GPV_RESPONSE new_data = parse_gpv_data(x) except (requests.RequestException, ValueError, OSError) as e: LOG.error(f'Error getting GPV data: {safe_error(e)}') return STATUS_ERROR_GPV_RESPONSE for el in new_data: update_gpv_data(PARSED_GPV_DATAS, el) remove_old_gpv_data(PARSED_GPV_DATAS) await check_and_send_gpv_message() return STATUS_SEND_OK async def check_and_send_gpv_message(): now_ts = datetime.now().timestamp() for el in list(PARSED_GPV_DATAS): if el.need_to_send and (el.actual_till_ts + 3600.0 >= now_ts): message = get_GPV_message(el.approved_from, el.event_date, el.hours_off, el.hours_on, el.outage_queue) if await asyncio.to_thread(notify, message): el.set_send() def update_gpv_data(data_list: List[GPVData], new_data: GPVData): # find by event date and queue old_data = next((obj for obj in data_list if obj.key == new_data.key), None) if old_data is None: data_list.append(new_data) LOG.debug(f'New element appended to list {new_data}') elif old_data.need_to_send: # notification is not sent yet - just take the latest data data_list[data_list.index(old_data)] = new_data elif not old_data.is_same_schedule(new_data): # schedule is changed after notification - it will be sent again data_list[data_list.index(old_data)] = new_data LOG.debug(f'Updated element {old_data} in list with new data {new_data}') else: LOG.debug(f'Element {old_data} was not updated due to the same') def remove_old_gpv_data(data_list: List[GPVData]): ts = datetime.now().timestamp() for el in data_list[:]: if el.actual_till_ts is None or (el.actual_till_ts + 1800) < ts: LOG.debug(f'Old element removed from list {el}') data_list.remove(el) def get_message_with_po(el: POData): address = html.escape(get_address(el.city, el.street, el.house)) if el.need_to_send: message = ( f'\U000026A0 Увага!\nНа {html.escape(el.start_date)} за адресою {address} заплановано ' f'{html.escape(el.po_type)} відключення (заявка від {html.escape(el.placement_date)}).\n' f'Причина відключення: {html.escape(el.reason)}\n' f'Початок вимкнення {html.escape(el.start_date)} об {html.escape(el.start_time)}\n' f'Кінець вимкнення {html.escape(el.stop_date)} об {html.escape(el.stop_time)}') else: message = f'На {html.escape(el.start_date)} Планових вимкнень немає \U0001F44C \U0001F483' return message def fetch_po_root(): if cfg.IS_DEBUG: with open(os.path.join(cfg.TEST_DIR, 'data_po_2days_02.html'), 'r', encoding='utf-8') as f: page = f.read() else: response = get_PO_response() if response.status_code != 200: LOG.error(f'PO response code={response.status_code}') return None page = response.text if not page.strip(): LOG.error('Empty PO page') return None return etree.fromstring(page, etree.HTMLParser()) async def process_po(): try: root = await asyncio.to_thread(fetch_po_root) if root is None: return STATUS_ERROR_PO_RESPONSE new_data = parse_po_data(root) except (requests.RequestException, etree.LxmlError, ValueError, OSError) as e: LOG.error(f'Error getting PO data: {safe_error(e)}') return STATUS_ERROR_PO_RESPONSE for el in new_data: update_po_data(PARSED_PO_DATAS, el) remove_old_po_data(PARSED_PO_DATAS) await check_and_send_po_message() return STATUS_SEND_OK async def check_and_send_po_message(): for el in list(PARSED_PO_DATAS): if el.need_to_send: message = get_message_with_po(el) if await asyncio.to_thread(notify, message): el.set_sent() el.set_notified_now() def get_daily_reminder_time(now: datetime) -> datetime: """Today's moment of the first data processing - notifications are repeated after it""" first_minute = min(cfg.PROCESSING_MINUTES) if cfg.PROCESSING_MINUTES else 0 return now.replace(hour=cfg.PROCESS_START_HOUR, minute=first_minute, second=0, microsecond=0) def is_reminder_needed(old_data: POData, now: Optional[datetime] = None) -> bool: now = now or datetime.now() if old_data.notified_at is None: return True # re-send notification every day if old_data.notified_at + timedelta(days=1) <= now: return True # or if the first data processing of the day is passed after the last notification reminder_time = get_daily_reminder_time(now) return old_data.notified_at < reminder_time <= now def update_po_data(data_list: List[POData], new_data: POData): # find by start TS and address old_data = next((obj for obj in data_list if obj.key == new_data.key), None) if old_data is None: data_list.append(new_data) LOG.debug(f'New element appended to list {new_data}') elif old_data.need_to_send: # notification is not sent yet - just take the latest data data_list[data_list.index(old_data)] = new_data elif old_data != new_data or is_reminder_needed(old_data): # data changed or it is time to remind - notification will be sent again data_list[data_list.index(old_data)] = new_data LOG.debug(f'Updated element {old_data} in list with new data {new_data}') else: LOG.debug(f'Element {old_data} was not updated due to the same') def remove_old_po_data(data_list: List[POData]): ts = datetime.now().timestamp() for el in data_list[:]: if el.start_ts and (el.start_ts + 3600) <= ts: LOG.debug(f'Old element removed from list {el}') data_list.remove(el) if __name__ == '__main__': pass