docs: improve SEO by adding sitemap (#435)

* add sphinx-sitemap * update sitemap version * add robots.txt file --------- Co-authored-by: Adrien Barbaresi <[email protected]>
adbar · Oct 30, 2023 · 2aa3197 · 2aa3197
1 parent 2c3b751
commit 2aa3197
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 1 deletion.
diff --git a/docs/conf.py b/docs/conf.py
@@ -52,7 +52,8 @@
     'sphinx.ext.autodoc',
     'sphinx.ext.intersphinx',
     'sphinx.ext.napoleon',
-    'sphinx.ext.viewcode'
+    'sphinx.ext.viewcode',
+    'sphinx_sitemap'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -135,3 +136,8 @@
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3/", None),
 }
+
+html_baseurl = 'https://trafilatura.readthedocs.io/'
+sitemap_url_scheme = "{lang}latest/{link}"
+
+html_extra_path = ['robots.txt']
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -4,3 +4,4 @@ pydata-sphinx-theme>=0.14.1
 docutils>=0.20.1
 # without version specifier
 trafilatura
+sphinx-sitemap
diff --git a/docs/robots.txt b/docs/robots.txt
@@ -0,0 +1,3 @@
+User-agent: *
+
+Sitemap: https://trafilatura.readthedocs.io/en/latest/sitemap.xml