Merge pull request DSpace#2710 from DSpace/bram-atmire-patch-1-2709

Update robots.txt.ejs
uniba-ub · Apr 29, 2024 · a8f65ce · a8f65ce
2 parents 13dbd6b + fbd3529
commit a8f65ce
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs
@@ -16,6 +16,8 @@ Disallow: /submit
 Disallow: /workspaceitems
 Disallow: /profile
 Disallow: /workflowitems
+# Crawlers should be able to access entity pages, but not the facet search links present on entity pages
+Disallow: /entities/*?f
 
 # Optionally uncomment the following line ONLY if sitemaps are working
 # and you have verified that your site is being indexed correctly.