From 2aa31979863f9642abd2d9182beb748c041eae62 Mon Sep 17 00:00:00 2001 From: Tony Yang Date: Mon, 30 Oct 2023 03:35:18 -0700 Subject: [PATCH] docs: improve SEO by adding sitemap (#435) * add sphinx-sitemap * update sitemap version * add robots.txt file --------- Co-authored-by: Adrien Barbaresi --- docs/conf.py | 8 +++++++- docs/requirements.txt | 1 + docs/robots.txt | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 docs/robots.txt diff --git a/docs/conf.py b/docs/conf.py index f32d737b..33c98402 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,7 +52,8 @@ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode' + 'sphinx.ext.viewcode', + 'sphinx_sitemap' ] # Add any paths that contain templates here, relative to this directory. @@ -135,3 +136,8 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), } + +html_baseurl = 'https://trafilatura.readthedocs.io/' +sitemap_url_scheme = "{lang}latest/{link}" + +html_extra_path = ['robots.txt'] diff --git a/docs/requirements.txt b/docs/requirements.txt index cde3357c..24f72038 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,3 +4,4 @@ pydata-sphinx-theme>=0.14.1 docutils>=0.20.1 # without version specifier trafilatura +sphinx-sitemap diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 00000000..15342021 --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,3 @@ +User-agent: * + +Sitemap: https://trafilatura.readthedocs.io/en/latest/sitemap.xml