import os import sys import traceback from datetime import datetime, time import pause import requests from application import Application import sqlite3 from selenium import webdriver from weeklyList import WeeklyList from workingHours import is_working_hours, next_working_hour, potential_midday_upload refresh_rate_minutes = 5 search_past_week = 0 search_num_weeks = 1 reset_table = False web_opts = webdriver.ChromeOptions() web_opts.add_argument('--no-sandbox') web_opts.add_argument('--headless') web_opts.add_argument('--disable-dev-shm-usage') def notify(message): api_url = 'https://hass.jennett-wheeler.co.uk/api/webhook/-Qx6jHsGLHwbBlJpLek5Nj8qS' requests.post(api_url, json={"title": "Monitor Planning", "message": message}) def update_other_applications(): there_were_newly_decided_applications = False with sqlite3.connect("./database.db") as _conn: _cursor = _conn.cursor() with webdriver.Chrome(options=web_opts) as _browser: print("Scrape Weekly List(s)") weekly_list = WeeklyList(_cursor) for search_week_idx in range(search_past_week, min(search_past_week + search_num_weeks, 9)): # Council only allow latest 9 weeks weekly_list.scrape(_browser, search_week_idx) there_were_newly_decided_applications = len(weekly_list.new_applications) > 0 print(" Number of new decided applications: " + str(len(weekly_list.new_applications))) print(" Number of existing applications: " + str(len(weekly_list.existing_applications))) print("") _cursor.execute("SELECT reference FROM applications WHERE caseOfficer IS NULL") newly_decided_applications = _cursor.fetchall() chris_decisions = 0 if len(newly_decided_applications) > 0: print(f"Scrape Newly Decided Applications: {len(newly_decided_applications)}") for (application_ref, ) in newly_decided_applications: _app = Application(_cursor, application_ref) _app.scrape_portal(_browser) if _app.caseOfficer == "Christopher Masters": chris_decisions += 1 print("") if there_were_newly_decided_applications: notify(f"Council has uploaded {len(weekly_list.new_applications)} new decisions ({chris_decisions} by Chris)") return there_were_newly_decided_applications if __name__ == '__main__': notify(f"Script Restarted") try: with sqlite3.connect("./database.db") as connection: cursor = connection.cursor() Application.CreateTableIfNotExists(cursor, reset_table) midday_checked = False while True: with sqlite3.connect("./database.db") as connection: application = Application(connection.cursor(), "25/00605/FUL") with webdriver.Chrome(options=web_opts) as browser: application.scrape_portal(browser, force=True, count_documents=True) if application.new_documents_found: notify(f"New Documents Found: Application now has {application.num_documents} documents") print("") if is_working_hours(): if not midday_checked and potential_midday_upload(): midday_checked = update_other_applications() if midday_checked: print(f"New decisions found at: {datetime.now().strftime('%H-%M-%S')}" ) pause.minutes(refresh_rate_minutes) else: if update_other_applications(): print(f"New decisions found at: {datetime.now().strftime('%H-%M-%S')}" ) next_start = next_working_hour() print(f"Pausing until: {next_start}") pause.until(next_start) else: if datetime.now().time() > time(19, 0, 0): midday_checked = False next_start = next_working_hour() print(f"Pausing until: {next_start}") pause.until(next_start) else: pause.minutes(refresh_rate_minutes) except KeyboardInterrupt: print('Interrupted') try: sys.exit(130) except SystemExit: os._exit(130) except Exception as e: print(f'Error found: {repr(e)}') print(traceback.format_exc()) notify(f"Error in planning monitor: {repr(e)}") try: sys.exit(130) except SystemExit: os._exit(130)