diff --git a/backend/src/main.rs b/backend/src/main.rs index 381bded..7212102 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -198,6 +198,7 @@ async fn main() -> Result<(), Box> { .route("/", get(home::get_home_view)) .route("/sitemap.xml", get(robots::get_sitemap_xml)) .route("/index.xml", get(robots::get_rss_xml)) + .route("/robots.txt", get(robots::get_robots_txt)) .route("/page/:id", get(home::get_page_view)) .route("/post/:id", get(post::get_post_view)) .route("/font/:id", get(fonts::get_font)) diff --git a/backend/src/robots.rs b/backend/src/robots.rs index c92f203..cfeb99c 100644 --- a/backend/src/robots.rs +++ b/backend/src/robots.rs @@ -127,3 +127,13 @@ pub async fn get_rss_xml(mut tx: Tx) -> (StatusCode, String) { (StatusCode::OK, RSS_XML.read().await.clone()) } + +pub async fn get_robots_txt() -> &'static str { + " + User-agent: GPTBot + Disallow: / + + User-agent: Google-Extended + Disallow: / + " +}