152 lines
5.7 KiB
Python
152 lines
5.7 KiB
Python
import os
|
|
import sys
|
|
import traceback
|
|
from datetime import datetime, time
|
|
|
|
import pause
|
|
import requests
|
|
|
|
from application import Application
|
|
|
|
import sqlite3
|
|
from selenium import webdriver
|
|
|
|
from weeklyList import WeeklyList
|
|
from workingHours import is_working_hours, next_working_hour, potential_midday_upload
|
|
|
|
refresh_rate_minutes = 5
|
|
search_past_week = 0
|
|
search_num_weeks = 1
|
|
reset_table = False
|
|
|
|
web_opts = webdriver.ChromeOptions()
|
|
web_opts.add_argument('--no-sandbox')
|
|
web_opts.add_argument('--headless')
|
|
web_opts.add_argument('--disable-dev-shm-usage')
|
|
|
|
def notify(message):
|
|
api_url = 'https://hass.jennett-wheeler.co.uk/api/webhook/-Qx6jHsGLHwbBlJpLek5Nj8qS'
|
|
requests.post(api_url, json={"title": "Monitor Planning", "message": message})
|
|
|
|
def notify_status(message):
|
|
api_url = 'https://hass.jennett-wheeler.co.uk/api/webhook/-Qx6jHsGLHwbBlJpLek5Nj8qS'
|
|
requests.post(api_url, json={"title": "Planning Script", "message": message})
|
|
|
|
def update_other_applications():
|
|
print("Scrape Weekly List(s)")
|
|
|
|
there_were_newly_decided_applications = False
|
|
with sqlite3.connect("./database.db") as _conn:
|
|
_cursor = _conn.cursor()
|
|
|
|
with webdriver.Chrome(options=web_opts) as _browser:
|
|
weekly_list = WeeklyList(_cursor)
|
|
|
|
for search_week_idx in range(search_past_week,
|
|
min(search_past_week + search_num_weeks, 9)): # Council only allow latest 9 weeks
|
|
try:
|
|
weekly_list.scrape(_browser, search_week_idx)
|
|
except Exception as e:
|
|
print(f'Error found: {repr(e)}')
|
|
print(traceback.format_exc())
|
|
print("Try again")
|
|
weekly_list.scrape(_browser, search_week_idx)
|
|
|
|
there_were_newly_decided_applications = len(weekly_list.new_applications) > 0
|
|
print(" Number of new decided applications: " + str(len(weekly_list.new_applications)))
|
|
print(" Number of existing applications: " + str(len(weekly_list.existing_applications)))
|
|
print("")
|
|
|
|
_cursor.execute("SELECT reference FROM applications WHERE caseOfficer IS NULL")
|
|
newly_decided_applications = _cursor.fetchall()
|
|
|
|
chris_decisions = 0
|
|
if len(newly_decided_applications) > 0:
|
|
print(f"Scrape Newly Decided Applications: {len(newly_decided_applications)}")
|
|
|
|
for (application_ref, ) in newly_decided_applications:
|
|
_app = Application(_cursor, application_ref)
|
|
try:
|
|
_app.scrape_portal(_browser)
|
|
except Exception as e:
|
|
print(f'Error found: {repr(e)}')
|
|
print(traceback.format_exc())
|
|
print("Try again")
|
|
_app.scrape_portal(_browser)
|
|
|
|
if _app.caseOfficer == "Christopher Masters":
|
|
chris_decisions += 1
|
|
|
|
print("")
|
|
|
|
if there_were_newly_decided_applications:
|
|
notify(f"Council has uploaded {len(weekly_list.new_applications)} new decisions ({chris_decisions} by Chris)")
|
|
|
|
return there_were_newly_decided_applications
|
|
|
|
if __name__ == '__main__':
|
|
notify_status(f"Script Started")
|
|
try:
|
|
with sqlite3.connect("./database.db") as connection:
|
|
cursor = connection.cursor()
|
|
Application.CreateTableIfNotExists(cursor, reset_table)
|
|
|
|
midday_checked = False
|
|
while True:
|
|
with sqlite3.connect("./database.db") as connection:
|
|
application = Application(connection.cursor(), "25/00605/FUL")
|
|
|
|
with webdriver.Chrome(options=web_opts) as browser:
|
|
try:
|
|
application.scrape_portal(browser, force=True, count_documents=True)
|
|
except Exception as e:
|
|
print(f'Error found: {repr(e)}')
|
|
print(traceback.format_exc())
|
|
print("Try again")
|
|
application.scrape_portal(browser, force=True, count_documents=True)
|
|
|
|
if application.new_documents_found:
|
|
notify(f"New Documents Found: Application now has {application.num_documents} documents")
|
|
print("")
|
|
|
|
if is_working_hours():
|
|
if not midday_checked and potential_midday_upload():
|
|
midday_checked = update_other_applications()
|
|
if midday_checked:
|
|
print(f"New decisions found at: {datetime.now().strftime('%H-%M-%S')}" )
|
|
|
|
pause.minutes(refresh_rate_minutes)
|
|
else:
|
|
if update_other_applications():
|
|
print(f"New decisions found at: {datetime.now().strftime('%H-%M-%S')}" )
|
|
|
|
next_start = next_working_hour()
|
|
print(f"Pausing until: {next_start}")
|
|
pause.until(next_start)
|
|
else:
|
|
if datetime.now().time() > time(19, 0, 0):
|
|
midday_checked = False
|
|
next_start = next_working_hour()
|
|
print(f"Pausing until: {next_start}")
|
|
pause.until(next_start)
|
|
|
|
else:
|
|
pause.minutes(refresh_rate_minutes)
|
|
|
|
|
|
except KeyboardInterrupt:
|
|
print('Interrupted')
|
|
try:
|
|
sys.exit(130)
|
|
except SystemExit:
|
|
os._exit(130)
|
|
|
|
except Exception as e:
|
|
print(f'Error found: {repr(e)}')
|
|
print(traceback.format_exc())
|
|
notify_status(f"Error in planning monitor: {repr(e)}")
|
|
|
|
try:
|
|
sys.exit(130)
|
|
except SystemExit:
|
|
os._exit(130) |