Merge pull request #66 from open-sdg/robots-txt

Robots.txt files to disallow crawlers
open-sdg · Jul 20, 2023 · 20bd3b2 · 20bd3b2
2 parents 912cf47 + 436fbf1
commit 20bd3b2
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 0 deletions.
diff --git a/.github/workflows/deploy-to-production.yml b/.github/workflows/deploy-to-production.yml
@@ -49,6 +49,9 @@ jobs:
     - name: Build data
       run: |
         python scripts/build_data.py
+    - name: Place public files
+      run: |
+        cp public/robots-prod.txt _site/robots.txt
     - name: Deploy to GitHub Pages
       uses: JamesIves/[email protected]
       with:

diff --git a/.github/workflows/deploy-to-staging.yml b/.github/workflows/deploy-to-staging.yml
@@ -43,6 +43,9 @@ jobs:
     - name: Build data
       run: |
         python scripts/build_data.py
+    - name: Place public files
+      run: |
+        cp public/robots-staging.txt _site/robots.txt
     - name: Deploy to GitHub Pages
       uses: JamesIves/[email protected]
       with:

diff --git a/public/robots-prod.txt b/public/robots-prod.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/public/robots-staging.txt b/public/robots-staging.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /