import time
from sqlite3 import Cursor
import re
from selenium.webdriver.support.select import Select
from application import Application
from selenium.webdriver.chrome.webdriver import WebDriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait as Wait
timeout = 5
base_url = "https://app.bathnes.gov.uk/webforms/planning"
TAG_RE = re.compile(r'<[^>]+>')
class WeeklyList:
def __init__(self, cursor: Cursor):
Application.CreateTableIfNotExists(cursor)
self.cursor = cursor
self.new_applications = []
self.existing_applications = []
def scrape(self, browser: WebDriver, search_past_week = 0):
browser.refresh()
browser.get(f"{base_url}/search.html#weeklyList")
# Bring up list of decided applications
search_button = Wait(browser, timeout=timeout).until(EC.element_to_be_clickable((By.ID, "weeklySearchBtn")))
time.sleep(0.5) # Give a little extra time
search_type = Select(browser.find_element(by=By.ID, value="weeklyListOption"))
search_type.select_by_value('decided')
search_week = Select(browser.find_element(by=By.ID, value="weeklyListBetween"))
search_week.select_by_index(search_past_week)
week_str = search_week.options[search_past_week].text.split(" to ")[0]
print(f"Week: {week_str}")
search_button.click()
results = Wait(browser, timeout=timeout).until(EC.visibility_of_element_located((By.ID, "results-table")))
rows = results.find_elements(By.TAG_NAME, "tr")
for row in rows:
col = row.find_elements(By.TAG_NAME, "td")[0]
application_html = col.get_attribute('innerHTML').replace('\n', '
')
application_ref_html = application_html.strip().split("
")[0].strip()
application_ref = TAG_RE.sub('', application_ref_html).replace("Application Reference: ", "")
application = Application(self.cursor, application_ref)
if application.caseOfficer:
self.existing_applications.append(application)
else:
self.new_applications.append(application)