Exclude specific-version docs from search engines (#8016)

treeverse · Aug 5, 2024 · d40b90b · d40b90b
1 parent 7fdb5c2
commit d40b90b
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 1 deletion.
diff --git a/.github/workflows/docs-release.yaml b/.github/workflows/docs-release.yaml
@@ -35,7 +35,7 @@ jobs:
 
       - name: Build release
         working-directory: docs
-        run: bundle exec jekyll build --config _config.yml -d _site/${{ steps.version.outputs.tag }} -b /${{ steps.version.outputs.tag }}
+        run: bundle exec jekyll build --config _config.yml,_config.version.yml -d _site/${{ steps.version.outputs.tag }} -b /${{ steps.version.outputs.tag }}
 
       - name: Remove robots.txt
         working-directory: docs/_site/${{ steps.version.outputs.tag }}

diff --git a/docs/_config.version.yml b/docs/_config.version.yml
@@ -0,0 +1 @@
+robots_exclude_from_indexes: true
diff --git a/docs/_includes/head_custom.html b/docs/_includes/head_custom.html
@@ -6,6 +6,9 @@
         'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
     })(window,document,'script','dataLayer','GTM-KTZBZW9');</script>
 <!-- End Google Tag Manager Head -->
+{% if site.robots_exclude_from_indexes %}
+<meta name="robots" content="noindex">
+{% endif %}
 <meta property="og:image" content="{{ site.baseurl }}{{ site.image }}">
 <link rel="preload" href="/assets/fonts/Rene Bieder - Galano Grotesque.otf" as="font" type="font/otf" crossorigin>
 <link rel="preload" href="/assets/fonts/Rene Bieder - Galano Grotesque Medium.otf" as="font" type="font/otf" crossorigin>

diff --git a/docs/robots.txt b/docs/robots.txt
@@ -3,4 +3,8 @@ layout: null
 sitemap: false
 ---
 User-agent: *
+# blocking older versions of the API.
+# note: this prevents direct indexing and access, but these pages are still available via refs from other sites.
+Disallow: /v0.*
+Disallow: /v1.*
 Sitemap: {{ site.url }}{{ site.baseurl }}/sitemap.xml