Update url_generator to use EDGAR API URL

bellingcat · Sep 8, 2024 · c4e0163 · c4e0163
1 parent f3204b8
commit c4e0163
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 12 deletions.
diff --git a/edgar_tool/url_generator.py b/edgar_tool/url_generator.py
@@ -2,7 +2,7 @@
 from typing import Literal, TypedDict
 from urllib import parse
 
-from edgar_tool.constants import PEO_IN_AND_INC_IN_TO_SEC_FORM_ID
+from edgar_tool.constants import PEO_IN_AND_INC_IN_TO_SEC_FORM_ID, TEXT_SEARCH_BASE_URL
 
 
 class SearchQueryKwargs(TypedDict, total=False):
@@ -123,7 +123,6 @@ def filing_category(self):
 
 
 def generate_search_url_for_kwargs(search_kwargs: SearchQueryKwargs) -> str:
-    base_url = "https://www.sec.gov/edgar/search/#/"
     validated_params = _ValidSearchParams(**search_kwargs)
     query_params = {
         "q": validated_params.keywords,
@@ -158,4 +157,4 @@ def generate_search_url_for_kwargs(search_kwargs: SearchQueryKwargs) -> str:
     encoded_params = parse.urlencode(
         query_params, doseq=True, encoding="utf-8", quote_via=parse.quote
     )
-    return parse.urljoin(base=base_url, url=encoded_params, allow_fragments=False)
+    return TEXT_SEARCH_BASE_URL + encoded_params
diff --git a/tests/test_url_generator.py b/tests/test_url_generator.py
@@ -26,7 +26,7 @@ def test_should_correctly_generate_search_url_for_single_word():
     produces the correct search URL"""
     # GIVEN
     keywords = ["10-K"]
-    expected_url = "https://www.sec.gov/edgar/search/#/q=10-K"
+    expected_url = f"https://efts.sec.gov/LATEST/search-index?q=10-K"
 
     # WHEN
     actual_url = url_generator.generate_search_url_for_kwargs({"keywords": keywords})
@@ -39,7 +39,7 @@ def test_should_correctly_generate_search_url_for_exact_phrase():
     # GIVEN
     keywords = ["Insider trading report"]
     expected_url = (
-        "https://www.sec.gov/edgar/search/#/q=%22Insider%20trading%20report%22"
+        "https://efts.sec.gov/LATEST/search-index?q=%22Insider%20trading%20report%22"
     )
 
     # WHEN
@@ -122,9 +122,7 @@ def test_generates_correct_url_for_date_ranges(date_kwargs, url_ending):
     """Tests that various date range options are correctly translated
     into the seach URL."""
     # GIVEN
-    expected_url = (
-        f"https://www.sec.gov/edgar/search/#/q=%22Ford%20Motor%20Co%22{url_ending}"
-    )
+    expected_url = f"https://efts.sec.gov/LATEST/search-index?q=%22Ford%20Motor%20Co%22{url_ending}"
     test_kwargs = {**{"keywords": ["Ford Motor Co"]}, **date_kwargs}
 
     # WHEN
@@ -153,7 +151,7 @@ def test_generates_correct_url_for_date_ranges(date_kwargs, url_ending):
 )
 def test_generates_correct_url_for_filing_category(filing_category, url_ending):
     # GIVEN
-    expected_url = f"https://www.sec.gov/edgar/search/#/q=Ignore{url_ending}"
+    expected_url = f"https://efts.sec.gov/LATEST/search-index?q=Ignore{url_ending}"
     test_kwargs = {"keywords": ["Ignore"], "filing_category": filing_category}
 
     # WHEN
@@ -177,7 +175,7 @@ def test_generates_correct_url_for_filing_category(filing_category, url_ending):
 def test_generates_correct_url_for_single_forms(single_forms, url_ending):
     # GIVEN
     expected_url = (
-        f"https://www.sec.gov/edgar/search/#/q=Ignore&category=custom{url_ending}"
+        f"https://efts.sec.gov/LATEST/search-index?q=Ignore&category=custom{url_ending}"
     )
     test_kwargs = {"keywords": ["Ignore"], "single_forms": single_forms}
 
@@ -532,7 +530,7 @@ def test_should_correctly_generate_search_url_for_peo_in(
         self, abbreviation, expected_location_code
     ):
         # GIVEN
-        expected_url = f"https://www.sec.gov/edgar/search/#/q=a&locationCode={expected_location_code}"
+        expected_url = f"https://efts.sec.gov/LATEST/search-index?q=a&locationCode={expected_location_code}"
 
         # WHEN
         actual_url = url_generator.generate_search_url_for_kwargs(
@@ -546,7 +544,7 @@ def test_should_correctly_generate_search_url_for_inc_in(
         self, abbreviation, expected_location_code
     ):
         # GIVEN
-        expected_url = f"https://www.sec.gov/edgar/search/#/q=a&locationType=incorporated&locationCode={expected_location_code}"
+        expected_url = f"https://efts.sec.gov/LATEST/search-index?q=a&locationType=incorporated&locationCode={expected_location_code}"
 
         # WHEN
         actual_url = url_generator.generate_search_url_for_kwargs(