58 lines
2.2 KiB
Python
58 lines
2.2 KiB
Python
import time
|
|
from sqlite3 import Cursor
|
|
import re
|
|
|
|
from selenium.webdriver.support.select import Select
|
|
|
|
from application import Application
|
|
|
|
from selenium.webdriver.chrome.webdriver import WebDriver
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from selenium.webdriver.support.wait import WebDriverWait as Wait
|
|
|
|
timeout = 5
|
|
base_url = "https://app.bathnes.gov.uk/webforms/planning"
|
|
TAG_RE = re.compile(r'<[^>]+>')
|
|
|
|
class WeeklyList:
|
|
def __init__(self, cursor: Cursor):
|
|
Application.CreateTableIfNotExists(cursor)
|
|
self.cursor = cursor
|
|
self.new_applications = []
|
|
self.existing_applications = []
|
|
|
|
def scrape(self, browser: WebDriver, search_past_week = 0):
|
|
browser.refresh()
|
|
browser.get(f"{base_url}/search.html#weeklyList")
|
|
|
|
# Bring up list of decided applications
|
|
search_button = Wait(browser, timeout=timeout).until(EC.element_to_be_clickable((By.ID, "weeklySearchBtn")))
|
|
time.sleep(0.5) # Give a little extra time
|
|
|
|
search_type = Select(browser.find_element(by=By.ID, value="weeklyListOption"))
|
|
search_type.select_by_value('decided')
|
|
search_week = Select(browser.find_element(by=By.ID, value="weeklyListBetween"))
|
|
search_week.select_by_index(search_past_week)
|
|
|
|
week_str = search_week.options[search_past_week].text.split(" to ")[0]
|
|
print(f"Week: {week_str}")
|
|
|
|
search_button.click()
|
|
|
|
results = Wait(browser, timeout=timeout).until(EC.visibility_of_element_located((By.ID, "results-table")))
|
|
|
|
rows = results.find_elements(By.TAG_NAME, "tr")
|
|
for row in rows:
|
|
col = row.find_elements(By.TAG_NAME, "td")[0]
|
|
application_html = col.get_attribute('innerHTML').replace('\n', '<br>')
|
|
|
|
application_ref_html = application_html.strip().split("<br>")[0].strip()
|
|
application_ref = TAG_RE.sub('', application_ref_html).replace("Application Reference: ", "")
|
|
|
|
application = Application(self.cursor, application_ref)
|
|
if application.caseOfficer:
|
|
self.existing_applications.append(application)
|
|
else:
|
|
self.new_applications.append(application)
|