import time from sqlite3 import Cursor import re from selenium.webdriver.support.select import Select from application import Application from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait as Wait timeout = 5 base_url = "https://app.bathnes.gov.uk/webforms/planning" TAG_RE = re.compile(r'<[^>]+>') class WeeklyList: def __init__(self, cursor: Cursor): Application.CreateTableIfNotExists(cursor) self.cursor = cursor self.new_applications = [] self.existing_applications = [] def scrape(self, browser: WebDriver, search_past_week = 0): browser.refresh() browser.get(f"{base_url}/search.html#weeklyList") # Bring up list of decided applications search_button = Wait(browser, timeout=timeout).until(EC.element_to_be_clickable((By.ID, "weeklySearchBtn"))) time.sleep(0.5) # Give a little extra time search_type = Select(browser.find_element(by=By.ID, value="weeklyListOption")) search_type.select_by_value('decided') search_week = Select(browser.find_element(by=By.ID, value="weeklyListBetween")) search_week.select_by_index(search_past_week) week_str = search_week.options[search_past_week].text.split(" to ")[0] print(f"Week: {week_str}") search_button.click() results = Wait(browser, timeout=timeout).until(EC.visibility_of_element_located((By.ID, "results-table"))) rows = results.find_elements(By.TAG_NAME, "tr") for row in rows: col = row.find_elements(By.TAG_NAME, "td")[0] application_html = col.get_attribute('innerHTML').replace('\n', '
') application_ref_html = application_html.strip().split("
")[0].strip() application_ref = TAG_RE.sub('', application_ref_html).replace("Application Reference: ", "") application = Application(self.cursor, application_ref) if application.caseOfficer: self.existing_applications.append(application) else: self.new_applications.append(application)