fix: apply seo agent improvements to src/app/robots.ts

src/app/robots.ts

@@ -0,0 +1,32 @@
+import type { MetadataRoute } from 'next'
+
+const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL || 'https://example.com'
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        allow: '/',
+        disallow: [
+          '/admin/',
+          '/admin',
+          '/api/',
+        ],
+      },
+      {
+        // Block AI training crawlers
+        userAgent: [
+          'GPTBot',
+          'ChatGPT-User',
+          'CCBot',
+          'anthropic-ai',
+          'Claude-Web',
+        ],
+        disallow: '/',
+      },
+    ],
+    sitemap: `${SITE_URL}/sitemap.xml`,
+    host: SITE_URL,
+  }
+}