From d6a2aded33317584562bf2435e79a69ba3c545a2 Mon Sep 17 00:00:00 2001 From: Marc Jay <580744+marcjay@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:37:00 +0100 Subject: [PATCH] Fix Windsor and Maidenhead source (#2843) * Fix Windsor and Maidenhead source Council website appears to have changed, now can go directly to property with UPRN in the URL Fixes #2841 * Address code review comments and revert the removal of the postcode argument --- .../translations/de.json | 4 +- .../translations/en.json | 4 +- .../translations/it.json | 4 +- .../source/rbwm_gov_uk.py | 82 ++++++------------- doc/source/rbwm_gov_uk.md | 9 +- 5 files changed, 33 insertions(+), 70 deletions(-) diff --git a/custom_components/waste_collection_schedule/translations/de.json b/custom_components/waste_collection_schedule/translations/de.json index fefdfddab..140850868 100644 --- a/custom_components/waste_collection_schedule/translations/de.json +++ b/custom_components/waste_collection_schedule/translations/de.json @@ -15949,7 +15949,7 @@ "description": "Konfiguriere deinen Service Provider. Mehr details: https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/doc/source/rbwm_gov_uk.md", "data": { "calendar_title": "Kalender Titel", - "postcode": "PLZ", + "postcode": "(Leave Empty)", "uprn": "UPRN" }, "data_description": { @@ -15962,7 +15962,7 @@ "description": "Konfiguriere deinen Service Provider. Mehr details: https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/doc/source/rbwm_gov_uk.md", "data": { "calendar_title": "Kalender Titel", - "postcode": "PLZ", + "postcode": "(Leave Empty)", "uprn": "UPRN" }, "data_description": { diff --git a/custom_components/waste_collection_schedule/translations/en.json b/custom_components/waste_collection_schedule/translations/en.json index 6d8f69119..4b3b7d5bc 100644 --- a/custom_components/waste_collection_schedule/translations/en.json +++ b/custom_components/waste_collection_schedule/translations/en.json @@ -15959,7 +15959,7 @@ "description": "Configure your service provider. More details: https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/doc/source/rbwm_gov_uk.md.", "data": { "calendar_title": "Calendar Title", - "postcode": "Postcode", + "postcode": "(Leer lassen)", "uprn": "UPRN" }, "data_description": { @@ -15972,7 +15972,7 @@ "description": "Configure your service provider. More details: https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/doc/source/rbwm_gov_uk.md.", "data": { "calendar_title": "Calendar Title", - "postcode": "Postcode", + "postcode": "(Leer lassen)", "uprn": "UPRN" }, "data_description": { diff --git a/custom_components/waste_collection_schedule/translations/it.json b/custom_components/waste_collection_schedule/translations/it.json index 591b68445..8a96b0f39 100644 --- a/custom_components/waste_collection_schedule/translations/it.json +++ b/custom_components/waste_collection_schedule/translations/it.json @@ -15938,7 +15938,7 @@ "description": "Compila i campi per ottenere le informazioni sul tuo servizio di raccolta. Maggiori informazioni: https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/doc/source/rbwm_gov_uk.md.", "data": { "calendar_title": "Nome Calendario", - "postcode": "Codice Postale CAP", + "postcode": "(Lascia vuoto)", "uprn": "UPRN" }, "data_description": { @@ -15951,7 +15951,7 @@ "description": "Compila i campi per ottenere le informazioni sul tuo servizio di raccolta. Per maggiori informazioni: https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/doc/source/rbwm_gov_uk.md.", "data": { "calendar_title": "Nome Calendario", - "postcode": "Codice Postale CAP", + "postcode": "(Lascia vuoto)", "uprn": "UPRN" }, "data_description": { diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/rbwm_gov_uk.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/rbwm_gov_uk.py index 7acca7509..e9b284c80 100644 --- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/rbwm_gov_uk.py +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/rbwm_gov_uk.py @@ -1,8 +1,6 @@ -from datetime import datetime -from typing import Any - import requests -from bs4 import BeautifulSoup, Tag +from bs4 import BeautifulSoup +from dateutil.parser import parse from waste_collection_schedule import Collection # type: ignore[attr-defined] TITLE = "Windsor and Maidenhead" @@ -10,9 +8,9 @@ URL = "https://my.rbwm.gov.uk/" TEST_CASES = { "Windsor 1": {"postcode": "SL4 4EN", "uprn": 100080381393}, - "Windsor 2": {"postcode": "SL4 3NX", "uprn": "100080384194"}, - "Maidenhead 1": {"postcode": "SL6 8EH", "uprn": "100080359672"}, - "Maidenhead 2": {"postcode": "SL6 5HX", "uprn": 100080355442}, + "Windsor 2": {"postcode": "", "uprn": "100080384194"}, + "Maidenhead 1": {"uprn": "100080359672"}, + "Maidenhead 2": {"uprn": 100080355442}, } @@ -22,75 +20,45 @@ "recycling": "mdi:recycle", } +PARAM_TRANSLATIONS = { + "de": { + "postcode": "(Leave Empty)", + "uprn": "UPRN", + }, + "en": { + "postcode": "(Leer lassen)", + "uprn": "UPRN", + }, + "it": { + "postcode": "(Lascia vuoto)", + "uprn": "UPRN", + }, +} API_URL = "https://forms.rbwm.gov.uk/bincollections" -REQUEST_TEMPLATE = """-----------------------------{rand_id} -Content-Disposition: form-data; name="{name}" - -{value} -""" -REQUEST_TEMPLATE_END = """-----------------------------{rand_id}-- -""" - HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", - "Content-Type": "multipart/form-data; boundary=---------------------------{rand_id}", } -def construct_payload(data: dict[str, Any]) -> tuple[str, dict[str, str]]: - rand_id = datetime.now().strftime("%Y%m%d%H%M%S%f%f%H%m") - payload = "" - for key, value in data.items(): - payload += REQUEST_TEMPLATE.format(rand_id=rand_id, name=key, value=value) - payload += REQUEST_TEMPLATE_END.format(rand_id=rand_id) - headers = HEADERS.copy() - headers["Content-Type"] = headers["Content-Type"].format(rand_id=rand_id) - return payload, headers - - class Source: - def __init__(self, uprn: str | int, postcode: str): + # postcode was previously required by this source. This is no longer the case but argument kept for backwards compatibility + def __init__(self, uprn: str | int, postcode=None): self._uprn: str = str(uprn).zfill(12) - self._postcode: str = postcode - - def _get_payload_data(self, form: Tag) -> dict[str, str]: - form_data = {"next": "next"} - for i in form.find_all("input"): - if i.get("name") == "injectedParams": - continue - form_data[i.get("name")] = i.get("value") - if i.get("name").endswith("_0_0"): - form_data[i.get("name")] = self._postcode - form_data[i.get("name").replace("_0_0", "_1_0")] = self._uprn - return form_data def fetch(self): s = requests.Session() - r = s.get(API_URL) + r = s.get(API_URL, params={"uprn": self._uprn}) r.raise_for_status() - URL_BASE = "https://" + r.url.removeprefix("https://").split("/")[0] soup = BeautifulSoup(r.text, "html.parser") - form = soup.find("main").find("form") - next_url = form.get("action") - if next_url.startswith("/"): - next_url = URL_BASE + next_url - - if not isinstance(form, Tag): - raise Exception("Invalid response from API") - form_data = self._get_payload_data(form) - payload, headers = construct_payload(form_data) - - r = s.post(next_url, data=payload, headers=headers) - r.raise_for_status() - - soup = BeautifulSoup(r.text, "html.parser") table = soup.find("table") + if table is None: + raise Exception("No results found. UPRN may be incorrect.") entries = [] for tr in table.find_all("tr"): @@ -100,7 +68,7 @@ def fetch(self): bi_type = tds[0].text.split("Collection Service")[0].strip() date_string = tds[1].text.strip() - date = datetime.strptime(date_string, "%d/%m/%Y").date() + date = parse(date_string).date() icon = ICON_MAP.get(bi_type.lower()) # Collection icon entries.append(Collection(date=date, t=bi_type, icon=icon)) diff --git a/doc/source/rbwm_gov_uk.md b/doc/source/rbwm_gov_uk.md index 749e54189..7bb478367 100644 --- a/doc/source/rbwm_gov_uk.md +++ b/doc/source/rbwm_gov_uk.md @@ -10,17 +10,13 @@ waste_collection_schedule: - name: rbwm_gov_uk args: uprn: "UPRN" - postcode: "POSTCODE" ``` ### Configuration Variables -**uprn** +**uprn** *(String | Integer) (required)* -**postcode** -*(String) (required)* - ## Example ```yaml @@ -29,11 +25,10 @@ waste_collection_schedule: - name: rbwm_gov_uk args: uprn: "100080381393" - postcode: "SL4 4EN" ``` ## How to find your -Go to [https://my.rbwm.gov.uk/special/find-your-collection-dates](https://my.rbwm.gov.uk/special/find-your-collection-dates) and search for your address. When you see your upcoming collections your address bar should look something like this: `https://my.rbwm.gov.uk/special/your-collection-dates?uprn=100080381393&subdate=2023-07-16&addr=104%20St%20Andrews%20Crescent%20Windsor%20SL4%204EN`. The first argument (after `uprn=`) is your UPRN (`100080381393` in this example) +Go to [https://forms.rbwm.gov.uk/bincollections](https://forms.rbwm.gov.uk/bincollections) and search for your address. When you see your upcoming collections your address bar should look something like this: `https://forms.rbwm.gov.uk/bincollections?uprn=100080381393`. The value after `uprn=` is your UPRN (`100080381393` in this example) Another way to discover your Unique Property Reference Number (UPRN) is by going to and entering in your address details.