Fixed query parameters not being considered for robots.txt evaluation

terminal42 · Dec 15, 2022 · f73df70 · f73df70
1 parent 9e7deb6
commit f73df70
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 1 deletion.
diff --git a/src/Subscriber/RobotsSubscriber.php b/src/Subscriber/RobotsSubscriber.php
@@ -152,7 +152,7 @@ private function handleDisallowedByRobotsTxtTag(CrawlUri $crawlUri): void
         // Check if an URI is allowed by the robots.txt
         $inspector = new Inspector($robotsTxt, $this->escargot->getUserAgent());
 
-        if (!$inspector->isAllowed($crawlUri->getUri()->getPath())) {
+        if (!$inspector->isAllowed($this->getPathAndQuery($crawlUri->getUri()))) {
             $crawlUri->addTag(self::TAG_DISALLOWED_ROBOTS_TXT);
 
             $this->logWithCrawlUri(
@@ -273,4 +273,15 @@ private function extractUrisFromSitemap(CrawlUri $sitemapUri, string $content):
             $this->escargot->addUriToQueue($uri, $sitemapUri);
         }
     }
+
+    private function getPathAndQuery(UriInterface $uri): string
+    {
+        $path = $uri->getPath();
+
+        if ($query = $uri->getQuery()) {
+            $path .= '?'.$query;
+        }
+
+        return $path;
+    }
 }
diff --git a/tests/EscargotTest.php b/tests/EscargotTest.php
@@ -284,6 +284,17 @@ public function shouldRequest(CrawlUri $crawlUri): string
                     }
                 }
 
+                // Skip the links that are disallowed by robots.txt
+                if ($crawlUri->hasTag(RobotsSubscriber::TAG_DISALLOWED_ROBOTS_TXT)) {
+                    $this->logWithCrawlUri(
+                        $crawlUri,
+                        LogLevel::DEBUG,
+                        'Do not request because it was disallowed by the robots.txt.'
+                    );
+
+                    return SubscriberInterface::DECISION_NEGATIVE;
+                }
+
                 // Skip rel="nofollow" links
                 if ($crawlUri->hasTag(HtmlCrawlerSubscriber::TAG_REL_NOFOLLOW)) {
                     $this->logWithCrawlUri(

diff --git a/tests/Fixtures/scenario18/_description.txt b/tests/Fixtures/scenario18/_description.txt
@@ -0,0 +1 @@
+Test robots.txt with query parameters are considered correctly
diff --git a/tests/Fixtures/scenario18/_logs.txt b/tests/Fixtures/scenario18/_logs.txt
@@ -0,0 +1,3 @@
+[Terminal42\Escargot\Subscriber\RobotsSubscriber] [URI: https://www.terminal42.ch/foobar?with-a-query=true (Level: 1, Processed: yes, Found on: https://www.terminal42.ch/, Tags: disallowed-robots-txt)] Added the "disallowed-robots-txt" tag because of the robots.txt content.
+[class@anonymous:EscargotTest] [URI: https://www.terminal42.ch/foobar?with-a-query=true (Level: 1, Processed: yes, Found on: https://www.terminal42.ch/, Tags: disallowed-robots-txt)] Do not request because it was disallowed by the robots.txt.
+[Terminal42\Escargot\Escargot] Finished crawling! Sent 1 request(s).
diff --git a/tests/Fixtures/scenario18/_requests.txt b/tests/Fixtures/scenario18/_requests.txt
@@ -0,0 +1 @@
+Successful request! URI: https://www.terminal42.ch/ (Level: 0, Processed: yes, Found on: root, Tags: none).
diff --git a/tests/Fixtures/scenario18/robots.txt b/tests/Fixtures/scenario18/robots.txt
@@ -0,0 +1,7 @@
+https://www.terminal42.ch/robots.txt
+
+HTTP/2.0 200 OK
+content-type: text/plain; charset=UTF-8
+
+User-agent: *
+Disallow: /*?*
diff --git a/tests/Fixtures/scenario18/root.txt b/tests/Fixtures/scenario18/root.txt
@@ -0,0 +1,12 @@
+https://www.terminal42.ch/
+
+HTTP/2.0 200 OK
+content-type: text/html; charset=UTF-8
+
+<html>
+    <head>
+    </head>
+    <body>
+        <a href="https://www.terminal42.ch/foobar?with-a-query=true">Link</a>
+    </body>
+</html>