diff --git a/src/app/robots.ts b/src/app/robots.ts new file mode 100644 index 0000000..f42fc53 --- /dev/null +++ b/src/app/robots.ts @@ -0,0 +1,32 @@ +import type { MetadataRoute } from 'next' + +const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL || 'https://example.com' + +export default function robots(): MetadataRoute.Robots { + return { + rules: [ + { + userAgent: '*', + allow: '/', + disallow: [ + '/admin/', + '/admin', + '/api/', + ], + }, + { + // Block AI training crawlers + userAgent: [ + 'GPTBot', + 'ChatGPT-User', + 'CCBot', + 'anthropic-ai', + 'Claude-Web', + ], + disallow: '/', + }, + ], + sitemap: `${SITE_URL}/sitemap.xml`, + host: SITE_URL, + } +}