chore: fix web robots txt

yjl9903 · Oct 2, 2024 · 372f23d · 372f23d
1 parent e852aca
commit 372f23d
Showing 1 changed file with 18 additions and 1 deletion.
diff --git a/apps/frontend/web/app/entry.server.tsx b/apps/frontend/web/app/entry.server.tsx
@@ -17,7 +17,24 @@ import { APP_HOST } from '~build/env';
 
 const { isSitemapUrl, sitemap } = createSitemapGenerator({
   siteUrl: `https://${APP_HOST}`,
-  generateRobotsTxt: true
+  generateRobotsTxt: true,
+  robotsTxtOptions: {
+    policies: [
+      {
+        userAgent: '*',
+        allow: [
+          '/docs/',
+          '/anime/',
+          '/detail/',
+          '/resource/',
+          '/resources/1',
+          '/resources/2',
+          '/resources/3'
+        ],
+        disallow: ['/feed.xml', '/api/']
+      }
+    ]
+  }
 });
 
 export default async function handleRequest(