feat: add /robots.txt blocking sensitive paths from crawlers

Disallows: /api/, /health, /metrics, /ws/, /avatar/, /blob/,
/media/, /static/, /assets/

apps/app/src/main.rs

@@ -13,6 +13,7 @@ use observability::{
 use sea_orm::ConnectionTrait;
 use service::AppService;
 use session::config::{PersistentSession, SessionLifecycle, TtlExtensionPolicy};
+use api::robots;
 use session::storage::RedisClusterSessionStore;
 use session::SessionMiddleware;
 use std::task::{Context, Poll};
@@ -218,6 +219,7 @@ async fn main() -> anyhow::Result<()> {
             .app_data(web::Data::new(cache.clone()))
             .app_data(http_snapshot_data.clone())
             .app_data(prometheus_handle_data.clone())
+            .route("/robots.txt", web::get().to(robots::robots))
             .route("/health", web::get().to(health_check))
             .route("/metrics", web::get().to(prometheus_handler))
             .configure(api::route::init_routes)

libs/api/robots.rs

@@ -0,0 +1,20 @@
+use actix_web::HttpResponse;
+
+/// Serves robots.txt, blocking all sensitive paths from crawlers.
+pub async fn robots() -> HttpResponse {
+    HttpResponse::Ok()
+        .content_type("text/plain; charset=utf-8")
+        .body(
+            r#"User-agent: *
+Disallow: /api/
+Disallow: /health
+Disallow: /metrics
+Disallow: /ws/
+Disallow: /avatar/
+Disallow: /blob/
+Disallow: /media/
+Disallow: /static/
+Disallow: /assets/
+"#,
+        )
+}