Allow OpenAI

Emilia-Capital · Nov 1, 2024 · 939e9bc · 939e9bc
1 parent 33fed56
commit 939e9bc
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 16 deletions.
diff --git a/eco-friendly-robots-txt.php b/eco-friendly-robots-txt.php
@@ -2,6 +2,7 @@
 /**
  * Plugin Name: Eco-Friendly Robots.txt
  * Plugin URI: https://joost.blog/plugins/eco-friendly-robots-txt/
+ * GitHub Plugin URI: Emilia-Capital/eco-friendly-robots-txt
  * Description: Optimizes your site's robots.txt to reduce server load and CO2 footprint by blocking unnecessary crawlers while allowing major search engines and specific tools.
  * Version: 0.5
  * Author: Joost de Valk

diff --git a/src/class-plugin.php b/src/class-plugin.php
@@ -91,10 +91,11 @@ public function modify_robots_txt( $output, $site_public ) {
 		$robots_txt .= "User-agent: *\n";
 		$robots_txt .= "Disallow: /\n";
 
-		$robots_txt .= "\n# Below are the crawlers that are allowed to crawl this site.\n";
+		$robots_txt .= "\n# Below are the crawlers that _are_ allowed to crawl this site.\n";
 		$robots_txt .= "# Below that list, you'll find paths that are blocked, even for them,\n";
 		$robots_txt .= "# and then paths within those blocked paths that are allowed.\n";
-		foreach ( $this->allowed_spiders as $crawler ) {
+		foreach ( $this->allowed_spiders as $crawler => $description ) {
+			$robots_txt .= "# $description\n";
 			$robots_txt .= "User-agent: $crawler\n";
 			$robots_txt .= "Allow: /\n";
 			foreach ( $this->blocked_paths as $path ) {
@@ -137,20 +138,23 @@ private function get_allowed_spiders() {
 		return apply_filters(
 			'emilia/ecofriendly_robots/allowed_spiders',
 			[
-				'Applebot',             // Apple.
-				'ia_archiver',          // Archive.org.
-				'Baiduspider',          // Baidu.
-				'Bingbot',              // Bing.
-				'DuckDuckBot',          // DuckDuckGo.
-				'Googlebot',            // Google.
-				'AdsBot-Google',        // Google Ads.
-				'MediaPartners-Google', // Google AdSense.
-				'Yandex',               // Yandex.
-				'Slurp',                // Yahoo.
-				'FacebookExternalHit',  // Facebook.
-				'LinkedInBot',          // LinkedIn.
-				'WhatsApp',             // WhatsApp.
-				'Twitterbot',           // X.
+				'AdsBot-Google'        => 'Google Ads',
+				'Applebot'             => 'Apple',
+				'Baiduspider'          => 'Baidu',
+				'Bingbot'              => 'Bing',
+				'ChatGPT-User'         => 'ChatGPT user requests',
+				'DuckDuckBot'          => 'DuckDuckGo',
+				'FacebookExternalHit'  => 'Facebook',
+				'GPTBot'               => 'OpenAI models',
+				'Googlebot'            => 'Google',
+				'LinkedInBot'          => 'LinkedIn',
+				'MediaPartners-Google' => 'Google AdSense',
+				'OAI-SearchBot'        => 'OpenAI Search',
+				'Slurp'                => 'Yahoo',
+				'Twitterbot'           => 'X',
+				'WhatsApp'             => 'WhatsApp',
+				'Yandex'               => 'Yandex',
+				'ia_archiver'          => 'Archive.org',
 			]
 		);
 	}