import { defineConfig } from 'astro/config'; import discovery from '@astrojs/discovery'; // Example configuration showing all available options export default defineConfig({ site: 'https://example.com', integrations: [ discovery({ // Robots.txt configuration robots: { crawlDelay: 2, allowAllBots: true, llmBots: { enabled: true, // Default bots are included, add custom ones here agents: [ 'Anthropic-AI', 'Claude-Web', 'GPTBot', 'ChatGPT-User', 'CustomBot', ], }, additionalAgents: [ { userAgent: 'BadBot', disallow: ['/'], }, { userAgent: 'GoodBot', allow: ['/api'], disallow: ['/admin'], }, ], customRules: ` # Custom rules User-agent: SpecialBot Crawl-delay: 10 `.trim(), }, // LLMs.txt configuration llms: { description: 'Your site description for AI assistants', keyFeatures: [ 'Feature 1', 'Feature 2', 'Feature 3', ], importantPages: [ { name: 'Documentation', path: '/docs', description: 'Complete API documentation', }, { name: 'Blog', path: '/blog', description: 'Latest articles and tutorials', }, ], instructions: ` When helping users with our site: 1. Check documentation first at /docs 2. Use provided API endpoints 3. Follow brand guidelines 4. Be helpful and accurate `.trim(), apiEndpoints: [ { path: '/api/chat', method: 'POST', description: 'Chat endpoint for conversations', }, { path: '/api/search', method: 'GET', description: 'Search API for content', }, ], techStack: { frontend: ['Astro', 'TypeScript', 'React'], backend: ['Node.js', 'FastAPI'], ai: ['Claude', 'GPT-4'], other: ['Docker', 'PostgreSQL'], }, brandVoice: [ 'Professional yet friendly', 'Technical but accessible', 'Focus on practical examples', ], customSections: { 'Contact': 'For support, email support@example.com', }, }, // Humans.txt configuration humans: { team: [ { name: 'Jane Doe', role: 'Creator & Developer', contact: 'jane@example.com', location: 'San Francisco, CA', twitter: '@janedoe', github: 'janedoe', }, { name: 'John Smith', role: 'Designer', contact: 'john@example.com', location: 'New York, NY', }, ], thanks: [ 'The Astro team for amazing tools', 'Open source community', 'Coffee ☕', ], site: { lastUpdate: 'auto', // or specific date like '2025-11-03' language: 'English', doctype: 'HTML5', ide: 'VS Code', techStack: ['Astro', 'TypeScript', 'React', 'Tailwind CSS'], standards: ['HTML5', 'CSS3', 'ES2022'], components: ['Astro Components', 'React Components'], software: ['Node.js', 'TypeScript', 'Git'], }, story: ` This project started when we realized there was a need for better discovery mechanisms on the web. We wanted to make it easy for search engines, AI assistants, and humans to understand what our site is about and how to interact with it. `.trim(), funFacts: [ 'Built with love and coffee', 'Over 100 commits in the first week', 'Designed with accessibility in mind', ], philosophy: [ 'Make the web more discoverable', 'Embrace open standards', 'Build with the future in mind', ], customSections: { 'SUSTAINABILITY': 'This site is carbon neutral and hosted on green servers.', }, }, // Sitemap configuration (passed to @astrojs/sitemap) sitemap: { filter: (page) => !page.includes('/admin') && !page.includes('/draft') && !page.includes('/private'), changefreq: 'weekly', priority: 0.7, }, // HTTP caching configuration (in seconds) caching: { robots: 3600, // 1 hour llms: 3600, // 1 hour humans: 86400, // 24 hours sitemap: 3600, // 1 hour }, // Custom templates (optional) // templates: { // robots: (config, siteURL) => `Your custom robots.txt content`, // llms: (config, siteURL) => `Your custom llms.txt content`, // humans: (config, siteURL) => `Your custom humans.txt content`, // }, }), ], });