From da9e96f6ddc414ad752e83b2c5ba9f7af73030c4 Mon Sep 17 00:00:00 2001
From: ZhenYi <434836402@qq.com>
Date: Sat, 25 Apr 2026 23:49:50 +0800
Subject: [PATCH] feat: add /robots.txt blocking sensitive paths from crawlers

Disallows: /api/, /health, /metrics, /ws/, /avatar/, /blob/,
/media/, /static/, /assets/
---
 apps/app/src/main.rs |  2 ++
 libs/api/robots.rs   | 20 ++++++++++++++++++++
 libs/api/sidemap.rs  |  0
 3 files changed, 22 insertions(+)
 create mode 100644 libs/api/sidemap.rs

diff --git a/apps/app/src/main.rs b/apps/app/src/main.rs
index 5693588..0ff0955 100644
--- a/apps/app/src/main.rs
+++ b/apps/app/src/main.rs
@@ -13,6 +13,7 @@ use observability::{
 use sea_orm::ConnectionTrait;
 use service::AppService;
 use session::config::{PersistentSession, SessionLifecycle, TtlExtensionPolicy};
+use api::robots;
 use session::storage::RedisClusterSessionStore;
 use session::SessionMiddleware;
 use std::task::{Context, Poll};
@@ -218,6 +219,7 @@ async fn main() -> anyhow::Result<()> {
             .app_data(web::Data::new(cache.clone()))
             .app_data(http_snapshot_data.clone())
             .app_data(prometheus_handle_data.clone())
+            .route("/robots.txt", web::get().to(robots::robots))
             .route("/health", web::get().to(health_check))
             .route("/metrics", web::get().to(prometheus_handler))
             .configure(api::route::init_routes)
diff --git a/libs/api/robots.rs b/libs/api/robots.rs
index e69de29..2eaef3e 100644
--- a/libs/api/robots.rs
+++ b/libs/api/robots.rs
@@ -0,0 +1,20 @@
+use actix_web::HttpResponse;
+
+/// Serves robots.txt, blocking all sensitive paths from crawlers.
+pub async fn robots() -> HttpResponse {
+    HttpResponse::Ok()
+        .content_type("text/plain; charset=utf-8")
+        .body(
+            r#"User-agent: *
+Disallow: /api/
+Disallow: /health
+Disallow: /metrics
+Disallow: /ws/
+Disallow: /avatar/
+Disallow: /blob/
+Disallow: /media/
+Disallow: /static/
+Disallow: /assets/
+"#,
+        )
+}
diff --git a/libs/api/sidemap.rs b/libs/api/sidemap.rs
new file mode 100644
index 0000000..e69de29