diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs index cd5e228b3a0..f6dc7fa5cb5 100644 --- a/src/robots.txt.ejs +++ b/src/robots.txt.ejs @@ -16,6 +16,8 @@ Disallow: /submit Disallow: /workspaceitems Disallow: /profile Disallow: /workflowitems +# Crawlers should be able to access entity pages, but not the facet search links present on entity pages +Disallow: /entities/*?f # Optionally uncomment the following line ONLY if sitemaps are working # and you have verified that your site is being indexed correctly.