fix: apply seo agent improvements to src/app/robots.ts

src/app/robots.ts

@@ -2,28 +2,25 @@ import type { MetadataRoute } from 'next'
 
 const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL || 'https://example.com'
 
+/**
+ * Next.js 14 robots.ts — automatically served at /robots.txt.
+ *
+ * Rules:
+ *  - All crawlers: allow public routes, disallow /admin/* and /api/*
+ *  - Googlebot: same rules, with specific sitemap pointer
+ */
 export default function robots(): MetadataRoute.Robots {
   return {
     rules: [
       {
         userAgent: '*',
         allow: '/',
-        disallow: [
+        disallow: ['/admin/', '/api/'],
-          '/admin/',
-          '/admin',
-          '/api/',
-        ],
       },
       {
-        // Block AI training crawlers
+        userAgent: 'Googlebot',
-        userAgent: [
+        allow: '/',
-          'GPTBot',
+        disallow: ['/admin/', '/api/'],
-          'ChatGPT-User',
-          'CCBot',
-          'anthropic-ai',
-          'Claude-Web',
-        ],
-        disallow: '/',
       },
     ],
     sitemap: `${SITE_URL}/sitemap.xml`,