EO_Sevice/utils.py

import asyncio
import html
import json
import os
import re
import uuid
from datetime import datetime, timedelta
from typing import List, Optional
from urllib.parse import quote

import requests
import urllib3
from lxml import etree

import app_config as cfg
from model import POData, GPVData, GPV_DATE_FORMAT

STATUS_SEND_OK = 0
STATUS_ERROR_SEND_TO_TG = 20
STATUS_ERROR_GPV_RESPONSE = 10
STATUS_ERROR_PO_RESPONSE = 30
STATUS_EMPTY_CONFIG = 5
STATUS_ERROR_SEND_TO_MX = 40

PARSED_PO_DATAS: List[POData] = []
PARSED_GPV_DATAS: List[GPVData] = []

LOG = cfg.LOG

# be-svitlo.oe.if.ua is requested without certificate verification
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

DIGIT_CHARS = [f'{d}\U000020E3' for d in range(10)]


def safe_error(e: Exception) -> str:
    """Error description without secrets (request URLs may contain the TG token)"""
    msg = f'{type(e).__name__}: {e}'
    for secret in (cfg.TG_TOKEN, cfg.MX_TOKEN):
        if secret:
            msg = msg.replace(secret, '***')
    return msg


def send_message_to_tg_get(msg: str) -> int:
    """Returns HTTP status code of Telegram response or 0 if the request failed"""
    if cfg.IS_DEBUG:
        print(f"Send to TG GET: {msg}")
        return 200
    if not cfg.TG_TOKEN or not cfg.TG_CHAT:
        LOG.error('Telegram is not configured')
        return 0
    try:
        response = requests.get(
            f"https://api.telegram.org/bot{cfg.TG_TOKEN}/sendMessage",
            params={'chat_id': cfg.TG_CHAT, 'parse_mode': 'html', 'text': msg},
            timeout=cfg.HTTP_TIMEOUT)
    except requests.RequestException as e:
        LOG.error(f'Error sending TG message: {safe_error(e)}')
        return 0
    if response.status_code != 200:
        LOG.error(f'TG response code={response.status_code}: {response.text[:500]}')
    return response.status_code


def send_message_to_tg_post(msg: str) -> int:
    """Returns HTTP status code of Telegram response or 0 if the request failed"""
    if cfg.IS_DEBUG:
        print(f"Send to TG POST: {msg}")
        return 200
    if not cfg.TG_TOKEN or not cfg.TG_CHAT:
        LOG.error('Telegram is not configured')
        return 0
    try:
        return requests.post(
            url=f'https://api.telegram.org/bot{cfg.TG_TOKEN}/sendMessage',
            data={'chat_id': cfg.TG_CHAT, 'text': msg, 'parse_mode': 'html'},
            timeout=cfg.HTTP_TIMEOUT
        ).status_code
    except requests.RequestException as e:
        LOG.error(f'Error sending TG message: {safe_error(e)}')
        return 0


def send_matrix_notification(message: str) -> bool:
    """Send notification to Matrix"""
    if not cfg.USE_MATRIX:
        return False

    if cfg.IS_DEBUG:
        print(f"Send to MX: {message}")
        return True

    if not (cfg.MX_SERVER and cfg.MX_ROOM and cfg.MX_TOKEN):
        LOG.error('Matrix is not configured')
        return False

    url = (f"{cfg.MX_SERVER}/_matrix/client/v3/rooms/{quote(cfg.MX_ROOM, safe='')}"
           f"/send/m.room.message/{uuid.uuid4().hex}")
    headers = {
        'Authorization': f'Bearer {cfg.MX_TOKEN}',
        'Content-Type': 'application/json'
    }
    payload = {
        'msgtype': 'm.text',
        'body': html.unescape(re.sub(r'<[^>]+>', '', message)),
        'format': 'org.matrix.custom.html',
        'formatted_body': message.replace('\n', '<br/>')
    }
    try:
        response = requests.put(url, headers=headers, json=payload, timeout=cfg.HTTP_TIMEOUT)
    except requests.RequestException as e:
        LOG.error(f"Error sending Matrix notification: {safe_error(e)}")
        return False
    if response.status_code != 200:
        LOG.error(f'MX response code={response.status_code}: {response.text[:500]}')
    return response.status_code == 200


def notify(message: str) -> bool:
    """Sends message to all configured messengers. Returns True if it was delivered at least to one of them"""
    delivered = False
    if send_message_to_tg_get(message) == 200:
        delivered = True
        LOG.debug(f'TG message send {message}')
    else:
        LOG.error(f'TG send error code: {STATUS_ERROR_SEND_TO_TG}')

    if cfg.USE_MATRIX:
        if send_matrix_notification(message):
            delivered = True
        else:
            LOG.error(f'MX send error code: {STATUS_ERROR_SEND_TO_MX}')
    return delivered


def get_GPV_response_from_oe(account):
    url = 'https://be-svitlo.oe.if.ua/schedule-by-search'
    headers = {
        "Host": "be-svitlo.oe.if.ua",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:144.0) Gecko/20100101 Firefox/144.0",
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "en-US,en;q=0.5",
        "Content-Type": "application/x-www-form-urlencoded;charset=utf-8",
        "Referer": "https://svitlo.oe.if.ua/",
        "Origin": "https://svitlo.oe.if.ua",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-site",
        "Connection": "keep-alive",
        "Priority": "u=0",
    }
    data = {
        "accountNumber": account,
        "userSearchChoice": "pob",
        "address": ""
    }
    return requests.post(url, headers=headers, data=data, verify=False, timeout=cfg.HTTP_TIMEOUT)


def get_PO_response():  # Power Outage
    utf8 = '✓'
    url = 'https://oe.if.ua/uk/shutdowns_table'
    params = {
        'utf8': utf8,
        'settlement': cfg.SETTLEMENT,
        'street': cfg.STREET,
        'house_number': cfg.HOUSE,
        'building_part_number': cfg.BUILDING_PART_NUMBER,
        'apartment': cfg.APARTMENT,
        'commit': 'Пошук'
    }
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36'
    }
    return requests.get(url=url, headers=headers, params=params, timeout=cfg.HTTP_TIMEOUT)


def has_numbers(in_str):
    return any(char.isdigit() for char in str(in_str))


def get_utf_chars(number, is_time=False):
    """Replaces ASCII digits with keycap emoji digits. is_time is kept for compatibility"""
    return ''.join(DIGIT_CHARS[int(c)] if c in '0123456789' else c for c in str(number))


def get_digit_message(arr, is_time=False):
    first = arr[0] if len(arr) > 0 else 0
    last = arr[1] if len(arr) > 1 else 0
    return f'з {get_utf_chars(first, is_time)} до {get_utf_chars(last, is_time)}'


def get_GPV_message(approved_from, event_date, hours_off, hours_on, outage_queue):
    queue = get_utf_chars(html.escape(str(outage_queue)))
    message = f'\U000026A1 <b>ГПВ на {html.escape(str(event_date))} для {queue} черги</b>'

    if len(hours_off) > 0:
        message += '\nВимкнення:'
        for hour_off, hour_on in zip(hours_off, hours_on):
            message += '\n' + get_digit_message([html.escape(str(hour_off)), html.escape(str(hour_on))], True)
    else:
        message += '\nВимкнень немає \U0001F44C \U0001F483'
    message += f'\n<b><i>{html.escape(str(approved_from))}</i></b>'
    return message


def get_address(city, street, house):
    return f'{city}, {street}, {house}'


def is_outage_date_before_or_same(stop_date):
    """True if the outage is not finished before today (or the date can't be parsed)"""
    try:
        parsed_date = datetime.strptime(stop_date, "%Y.%m.%d").date()
    except (TypeError, ValueError):
        LOG.warning(f'Wrong stop date: {stop_date!r}')
        return True
    return parsed_date >= datetime.now().date()


def get_row_text(row, path: str) -> str:
    return ' '.join(t.strip() for t in row.xpath(path) if t and t.strip())


def parse_po_data(root) -> List[POData]:
    res: List[POData] = []
    for row in root.xpath("//div[@class='table-row flex']"):
        po_data = POData(
            city=get_row_text(row, "./div[@class='city']//text()"),
            street=get_row_text(row, "./div[@class='street']//text()"),
            house=get_row_text(row, "./div[@class='house_number']//text()"),
            po_type=get_row_text(row, "./div[@class='type-shutdown']//text()"),
            reason=get_row_text(row, "./div[@class='reason-shutdown reason-text-container']/div[@class='reason-text-hide']//text()"),
            placement_date=get_row_text(row, "./div[@class='placement_date']/div[@class='date']//text()"),
            start_date=get_row_text(row, "./div[@class='shutdown_date']/div[@class='date']//text()"),
            stop_date=get_row_text(row, "./div[@class='turn_on_date']/div[@class='date']//text()"),
            start_time=get_row_text(row, "./div[@class='shutdown_date']/div[@class='time']//text()"),
            stop_time=get_row_text(row, "./div[@class='turn_on_date']/div[@class='time']//text()"))

        if not po_data.start_ts:
            LOG.warning(f'PO row skipped due to wrong start date/time: {po_data}')
            continue

        if not is_outage_date_before_or_same(po_data.stop_date):
            continue

        if po_data not in res:
            res.append(po_data)

    res.sort(key=lambda pd: pd.start_ts)

    return res


def load_last_datas(file_name):
    if os.path.isfile(file_name):
        with open(file_name, 'r', encoding='utf-8') as f:
            return f.read()
    else:
        return ''


def get_parsed_po_data():
    return PARSED_PO_DATAS


def get_parsed_gpv_data():
    return PARSED_GPV_DATAS


def gpv_date_key(entry: dict):
    try:
        return datetime.strptime(str(entry.get('eventDate')), GPV_DATE_FORMAT)
    except ValueError:
        return datetime.max


def parse_gpv_data(x) -> List[GPVData]:
    if not isinstance(x, dict):
        raise ValueError(f'Unexpected GPV response type: {type(x).__name__}')

    current = x.get('current') or {}
    if current.get('queue') is None:
        LOG.warning(f'No outage queue in GPV response: {current}')
        return []
    outage_queue = str(current['queue'])
    if current.get('subQueue'):
        outage_queue += '.' + str(current['subQueue'])

    schedule = x.get('schedule')
    if not isinstance(schedule, list):
        return []

    res: List[GPVData] = []
    for entry in sorted((e for e in schedule if isinstance(e, dict)), key=gpv_date_key):
        queues = entry.get('queues')
        event_date = entry.get('eventDate')
        if not isinstance(queues, dict) or not event_date:
            continue
        if outage_queue not in queues:
            LOG.warning(f'No queue {outage_queue} in GPV schedule for {event_date}')
            continue

        approved_since = entry.get('scheduleApprovedSince')
        approved_from = f'Запроваджено {approved_since}' if approved_since else 'Hемає даних про дату запровадження'

        hours_list = queues.get(outage_queue) or []
        if cfg.IS_DEBUG:
            print(f'hours_list{hours_list}, approved_from={approved_from}, event_date={event_date}')

        hours_off = []
        hours_on = []
        for h in hours_list:
            if isinstance(h, dict) and h.get("status") == 1 and h.get("from") and h.get("to"):
                hours_off.append(h["from"])
                hours_on.append(h["to"])

        if cfg.IS_DEBUG:
            print(f'hours_off = {hours_off}')
            print(f'hours_on = {hours_on}')

        gpv_data = GPVData(approved_from=approved_from, event_date=event_date, hours_off=hours_off,
                           hours_on=hours_on, outage_queue=outage_queue)
        if gpv_data.actual_till_ts is None:
            LOG.warning(f'GPV entry skipped due to wrong event date: {gpv_data}')
            continue
        res.append(gpv_data)
    return res


def fetch_gpv_json():
    if cfg.IS_DEBUG:
        with open(os.path.join(cfg.TEST_DIR, 'data_gpv.json'), 'r', encoding='utf-8') as f:
            return json.load(f)

    if not cfg.ACCOUNT:
        LOG.error('Account number is not configured')
        return None
    response = get_GPV_response_from_oe(cfg.ACCOUNT)
    if response.status_code != 200:
        LOG.error(f'GPV response code={response.status_code}')
        return None
    return response.json()


async def process_gpv():
    try:
        x = await asyncio.to_thread(fetch_gpv_json)
        if x is None:
            return STATUS_ERROR_GPV_RESPONSE
        new_data = parse_gpv_data(x)
    except (requests.RequestException, ValueError, OSError) as e:
        LOG.error(f'Error getting GPV data: {safe_error(e)}')
        return STATUS_ERROR_GPV_RESPONSE

    for el in new_data:
        update_gpv_data(PARSED_GPV_DATAS, el)
    remove_old_gpv_data(PARSED_GPV_DATAS)

    await check_and_send_gpv_message()
    return STATUS_SEND_OK


async def check_and_send_gpv_message():
    now_ts = datetime.now().timestamp()
    for el in list(PARSED_GPV_DATAS):
        if el.need_to_send and (el.actual_till_ts + 3600.0 >= now_ts):
            message = get_GPV_message(el.approved_from, el.event_date, el.hours_off, el.hours_on, el.outage_queue)
            if await asyncio.to_thread(notify, message):
                el.set_send()


def update_gpv_data(data_list: List[GPVData], new_data: GPVData):
    # find by event date and queue
    old_data = next((obj for obj in data_list if obj.key == new_data.key), None)
    if old_data is None:
        data_list.append(new_data)
        LOG.debug(f'New element appended to list {new_data}')
    elif old_data.need_to_send:
        # notification is not sent yet - just take the latest data
        data_list[data_list.index(old_data)] = new_data
    elif not old_data.is_same_schedule(new_data):
        # schedule is changed after notification - it will be sent again
        data_list[data_list.index(old_data)] = new_data
        LOG.debug(f'Updated element {old_data} in list with new data {new_data}')
    else:
        LOG.debug(f'Element {old_data} was not updated due to the same')


def remove_old_gpv_data(data_list: List[GPVData]):
    ts = datetime.now().timestamp()
    for el in data_list[:]:
        if el.actual_till_ts is None or (el.actual_till_ts + 1800) < ts:
            LOG.debug(f'Old element removed from list {el}')
            data_list.remove(el)


def get_message_with_po(el: POData):
    address = html.escape(get_address(el.city, el.street, el.house))
    if el.need_to_send:
        message = (
            f'\U000026A0 <b>Увага!</b>\nНа {html.escape(el.start_date)} за адресою {address} заплановано '
            f'<b>{html.escape(el.po_type)}</b> відключення (заявка від {html.escape(el.placement_date)}).\n'
            f'Причина відключення: {html.escape(el.reason)}\n'
            f'Початок вимкнення <b>{html.escape(el.start_date)} об {html.escape(el.start_time)}</b>\n'
            f'Кінець вимкнення  <b>{html.escape(el.stop_date)} об {html.escape(el.stop_time)}</b>')
    else:
        message = f'На {html.escape(el.start_date)} Планових вимкнень немає \U0001F44C \U0001F483'

    return message


def fetch_po_root():
    if cfg.IS_DEBUG:
        with open(os.path.join(cfg.TEST_DIR, 'data_po_2days_02.html'), 'r', encoding='utf-8') as f:
            page = f.read()
    else:
        response = get_PO_response()
        if response.status_code != 200:
            LOG.error(f'PO response code={response.status_code}')
            return None
        page = response.text
    if not page.strip():
        LOG.error('Empty PO page')
        return None
    return etree.fromstring(page, etree.HTMLParser())


async def process_po():
    try:
        root = await asyncio.to_thread(fetch_po_root)
        if root is None:
            return STATUS_ERROR_PO_RESPONSE
        new_data = parse_po_data(root)
    except (requests.RequestException, etree.LxmlError, ValueError, OSError) as e:
        LOG.error(f'Error getting PO data: {safe_error(e)}')
        return STATUS_ERROR_PO_RESPONSE

    for el in new_data:
        update_po_data(PARSED_PO_DATAS, el)
    remove_old_po_data(PARSED_PO_DATAS)

    await check_and_send_po_message()

    return STATUS_SEND_OK


async def check_and_send_po_message():
    for el in list(PARSED_PO_DATAS):
        if el.need_to_send:
            message = get_message_with_po(el)
            if await asyncio.to_thread(notify, message):
                el.set_sent()
                el.set_notified_now()


def get_daily_reminder_time(now: datetime) -> datetime:
    """Today's moment of the first data processing - notifications are repeated after it"""
    first_minute = min(cfg.PROCESSING_MINUTES) if cfg.PROCESSING_MINUTES else 0
    return now.replace(hour=cfg.PROCESS_START_HOUR, minute=first_minute, second=0, microsecond=0)


def is_reminder_needed(old_data: POData, now: Optional[datetime] = None) -> bool:
    now = now or datetime.now()
    if old_data.notified_at is None:
        return True
    # re-send notification every day
    if old_data.notified_at + timedelta(days=1) <= now:
        return True
    # or if the first data processing of the day is passed after the last notification
    reminder_time = get_daily_reminder_time(now)
    return old_data.notified_at < reminder_time <= now


def update_po_data(data_list: List[POData], new_data: POData):
    # find by start TS and address
    old_data = next((obj for obj in data_list if obj.key == new_data.key), None)
    if old_data is None:
        data_list.append(new_data)
        LOG.debug(f'New element appended to list {new_data}')
    elif old_data.need_to_send:
        # notification is not sent yet - just take the latest data
        data_list[data_list.index(old_data)] = new_data
    elif old_data != new_data or is_reminder_needed(old_data):
        # data changed or it is time to remind - notification will be sent again
        data_list[data_list.index(old_data)] = new_data
        LOG.debug(f'Updated element {old_data} in list with new data {new_data}')
    else:
        LOG.debug(f'Element {old_data} was not updated due to the same')


def remove_old_po_data(data_list: List[POData]):
    ts = datetime.now().timestamp()
    for el in data_list[:]:
        if el.start_ts and (el.start_ts + 3600) <= ts:
            LOG.debug(f'Old element removed from list {el}')
            data_list.remove(el)


if __name__ == '__main__':
    pass