From f8d4e10ffc1b53cbf9b4fe120cafb04e55dab9f2 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Sat, 8 Nov 2025 23:27:48 -0700 Subject: [PATCH] Complete all Tutorial and Getting Started documentation pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completed comprehensive, learning-oriented tutorial content following Diátaxis framework: Getting Started pages: - first-steps.md: Hands-on exploration of generated files with step-by-step verification - installation.md and quick-start.md: Already completed with clear installation paths Tutorial pages: - basic-setup.md: Progressive configuration from minimal to complete setup - configure-robots.md: Bot access control with practical examples - setup-llms.md: AI assistant context with instructions and API documentation - create-humans.md: Team credits, tech stack, and project story - security-canary.md: RFC 9116 security.txt and warrant canary implementation - webfinger.md: Federated discovery for ActivityPub and OpenID Connect All tutorials follow Diátaxis principles: - Step-by-step instructions that work for everyone - Concrete code examples with expected output - Build and verify commands after each step - Focus on getting learners started with confidence - Minimal explanation, maximum hands-on practice - Progressive complexity building on previous steps Documentation verified with successful docs build. --- .../docs/getting-started/first-steps.md | 197 ++++++- .../src/content/docs/tutorials/basic-setup.md | 365 +++++++++++- .../docs/tutorials/configure-robots.md | 405 ++++++++++++- .../content/docs/tutorials/create-humans.md | 508 +++++++++++++++- .../content/docs/tutorials/security-canary.md | 528 ++++++++++++++++- docs/src/content/docs/tutorials/setup-llms.md | 445 +++++++++++++- docs/src/content/docs/tutorials/webfinger.md | 547 +++++++++++++++++- status.json | 101 ++++ 8 files changed, 2964 insertions(+), 132 deletions(-) create mode 100644 status.json diff --git a/docs/src/content/docs/getting-started/first-steps.md b/docs/src/content/docs/getting-started/first-steps.md index 1534ad7..d17c2d1 100644 --- a/docs/src/content/docs/getting-started/first-steps.md +++ b/docs/src/content/docs/getting-started/first-steps.md @@ -3,29 +3,190 @@ title: First Steps description: Learn the basics of using @astrojs/discovery --- -This guide covers the fundamental concepts and first steps with @astrojs/discovery. +Now that we have @astrojs/discovery installed, let's explore what you've created and understand how it works. -:::note[Work in Progress] -This page is currently being developed. Check back soon for complete documentation. -::: +## What You Just Built -## Coming Soon +When you added the discovery integration to your Astro project, you enabled automatic generation of four essential discovery files. Let's see what each one does for your site. -This section will include: -- Detailed explanations -- Code examples -- Best practices -- Common patterns -- Troubleshooting tips +## Step 1: Build Your Site -## Related Pages +First, let's build the site to generate the discovery files: -- [Configuration Reference](/reference/configuration/) -- [API Reference](/reference/api/) -- [Examples](/examples/ecommerce/) +```bash +npm run build +``` -## Need Help? +You should see output indicating that your site has been built successfully. -- Check our [FAQ](/community/faq/) +## Step 2: Check the Generated Files + +Navigate to your `dist` folder. You'll find these new files: + +```bash +ls dist/ +``` + +You should see: +- `robots.txt` +- `llms.txt` +- `humans.txt` +- `sitemap-index.xml` + +Let's look at each one! + +## Step 3: Explore robots.txt + +Open `dist/robots.txt` in your text editor: + +```bash +cat dist/robots.txt +``` + +You'll see something like this: + +```txt +User-agent: * +Allow: / + +# Sitemaps +Sitemap: https://your-site.com/sitemap-index.xml + +# LLM-specific resources +User-agent: Anthropic-AI +User-agent: Claude-Web +User-agent: GPTBot +User-agent: ChatGPT-User +User-agent: cohere-ai +User-agent: Google-Extended +Allow: /llms.txt + +Crawl-delay: 1 +``` + +This file tells search engines and AI bots: +- All bots are allowed to crawl your site (`User-agent: *` with `Allow: /`) +- Where to find your sitemap +- AI bots can access `/llms.txt` for additional context +- To wait 1 second between requests (crawl delay) + +## Step 4: Explore llms.txt + +Now look at `dist/llms.txt`: + +```bash +cat dist/llms.txt +``` + +You'll see a structured file that helps AI assistants understand your site: + +```markdown +# your-site.com + +> Site built with Astro + +## Site Information +- Name: your-site.com +- URL: https://your-site.com + +## For AI Assistants + +This site is built with Astro and the @astrojs/discovery integration. + +## Tech Stack + +### Frontend +- Astro + +## Important Pages +- Home: https://your-site.com/ +``` + +This file provides context to AI assistants like Claude, helping them understand and reference your site correctly. + +## Step 5: Explore humans.txt + +Check `dist/humans.txt`: + +```bash +cat dist/humans.txt +``` + +You'll see credit information: + +```txt +/* SITE */ + +Last update: 2025-01-08 +Language: English +Doctype: HTML5 +Tech stack: Astro +``` + +This file credits the humans behind your site and documents your tech stack. + +## Step 6: View Your Sitemap + +Finally, look at `dist/sitemap-index.xml`: + +```bash +cat dist/sitemap-index.xml +``` + +You'll see an XML file listing all your pages, helping search engines index your site. + +## Step 7: Test in Development + +Now let's see these files in action during development: + +```bash +npm run dev +``` + +Once your dev server is running, open your browser and visit: + +- `http://localhost:4321/robots.txt` +- `http://localhost:4321/llms.txt` +- `http://localhost:4321/humans.txt` +- `http://localhost:4321/sitemap-index.xml` + +All these files are served dynamically! + +## What You've Learned + +You now know: +- How to build your site to generate discovery files +- What each discovery file contains +- How to view the files in both build and dev modes +- The purpose of each file + +## Next Steps + +Now that you understand the basics, let's customize these files: + +- [Basic Setup](/tutorials/basic-setup/) - Learn to customize the integration +- [Configure robots.txt](/tutorials/configure-robots/) - Control bot access +- [Setup llms.txt](/tutorials/setup-llms/) - Provide better AI context +- [Create humans.txt](/tutorials/create-humans/) - Credit your team + +## Troubleshooting + +### Files Not Showing Up? + +Make sure you have the `site` configured in `astro.config.mjs`: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', // This is required! + integrations: [discovery()] +}); +``` + +### Wrong URLs in Files? + +Check that your `site` URL matches your production domain. The integration uses this URL to generate absolute links. + +### Need More Help? + +- Check the [FAQ](/community/faq/) - Visit [Troubleshooting](/community/troubleshooting/) -- Open an issue on [GitHub](https://github.com/withastro/astro-discovery/issues) diff --git a/docs/src/content/docs/tutorials/basic-setup.md b/docs/src/content/docs/tutorials/basic-setup.md index 6d97b30..551c3ad 100644 --- a/docs/src/content/docs/tutorials/basic-setup.md +++ b/docs/src/content/docs/tutorials/basic-setup.md @@ -3,29 +3,356 @@ title: Basic Setup description: Set up @astrojs/discovery with default configuration --- -Learn how to set up @astrojs/discovery with the default configuration for immediate use. +In this tutorial, you'll learn how to customize the basic settings of @astrojs/discovery to match your project's needs. We'll start simple and gradually add more configuration. -:::note[Work in Progress] -This page is currently being developed. Check back soon for complete documentation. -::: +## What You'll Build -## Coming Soon +By the end of this tutorial, you'll have: +- A fully configured discovery integration +- Custom site description for AI assistants +- Team credits in humans.txt +- Properly configured robots.txt -This section will include: -- Detailed explanations -- Code examples -- Best practices -- Common patterns -- Troubleshooting tips +## Before You Start -## Related Pages +Make sure you have: +- @astrojs/discovery installed +- Your `site` URL configured in `astro.config.mjs` +- A working Astro project -- [Configuration Reference](/reference/configuration/) -- [API Reference](/reference/api/) -- [Examples](/examples/ecommerce/) +## Step 1: Start with the Minimal Setup -## Need Help? +Open your `astro.config.mjs` file. You should have something like this: -- Check our [FAQ](/community/faq/) -- Visit [Troubleshooting](/community/troubleshooting/) -- Open an issue on [GitHub](https://github.com/withastro/astro-discovery/issues) +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery() + ] +}); +``` + +This minimal setup works, but we can make it better! + +## Step 2: Add a Site Description + +Let's help AI assistants understand your site better. Add an `llms` configuration: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A personal blog about web development, focusing on modern JavaScript frameworks and best practices', + } + }) + ] +}); +``` + +Build your site and check `dist/llms.txt`: + +```bash +npm run build +cat dist/llms.txt +``` + +You'll now see your description in the generated file! + +## Step 3: Add Your Team Information + +Let's add credits to humans.txt. Update your configuration: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A personal blog about web development, focusing on modern JavaScript frameworks and best practices', + }, + humans: { + team: [ + { + name: 'Your Name', + role: 'Developer', + contact: 'you@example.com', + location: 'Your City', + } + ], + } + }) + ] +}); +``` + +Build again and check `dist/humans.txt`: + +```bash +npm run build +cat dist/humans.txt +``` + +You should see: + +```txt +/* TEAM */ + +Name: Your Name +Role: Developer +Contact: you@example.com +Location: Your City + +/* SITE */ + +Last update: 2025-01-08 +Language: English +Doctype: HTML5 +Tech stack: Astro +``` + +Great! Your name is now in the credits. + +## Step 4: Customize the Tech Stack + +Let's document the technologies you're actually using: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A personal blog about web development, focusing on modern JavaScript frameworks and best practices', + }, + humans: { + team: [ + { + name: 'Your Name', + role: 'Developer', + contact: 'you@example.com', + location: 'Your City', + } + ], + site: { + lastUpdate: 'auto', + language: 'English', + doctype: 'HTML5', + techStack: ['Astro', 'TypeScript', 'React', 'Tailwind CSS'], + } + } + }) + ] +}); +``` + +Build and verify: + +```bash +npm run build +cat dist/humans.txt +``` + +Now the tech stack section lists all your technologies! + +## Step 5: Add Key Features for AI + +Help AI assistants understand what makes your site special: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A personal blog about web development, focusing on modern JavaScript frameworks and best practices', + keyFeatures: [ + 'In-depth tutorials on modern web development', + 'Code examples with live demos', + 'Weekly newsletter with web dev tips', + 'Open source project showcase', + ], + }, + humans: { + team: [ + { + name: 'Your Name', + role: 'Developer', + contact: 'you@example.com', + location: 'Your City', + } + ], + site: { + lastUpdate: 'auto', + language: 'English', + doctype: 'HTML5', + techStack: ['Astro', 'TypeScript', 'React', 'Tailwind CSS'], + } + } + }) + ] +}); +``` + +Build and check `dist/llms.txt`: + +```bash +npm run build +cat dist/llms.txt +``` + +You'll see your key features listed prominently! + +## Step 6: Adjust Crawl Delay + +If you want to be more or less friendly to bots, adjust the crawl delay: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + crawlDelay: 2, // Wait 2 seconds between requests + }, + llms: { + description: 'A personal blog about web development, focusing on modern JavaScript frameworks and best practices', + keyFeatures: [ + 'In-depth tutorials on modern web development', + 'Code examples with live demos', + 'Weekly newsletter with web dev tips', + 'Open source project showcase', + ], + }, + humans: { + team: [ + { + name: 'Your Name', + role: 'Developer', + contact: 'you@example.com', + location: 'Your City', + } + ], + site: { + lastUpdate: 'auto', + language: 'English', + doctype: 'HTML5', + techStack: ['Astro', 'TypeScript', 'React', 'Tailwind CSS'], + } + } + }) + ] +}); +``` + +Check `dist/robots.txt`: + +```bash +npm run build +cat dist/robots.txt +``` + +The crawl delay is now 2 seconds! + +## Step 7: Test Everything + +Start your dev server and test all the files: + +```bash +npm run dev +``` + +Visit these URLs in your browser: +- `http://localhost:4321/robots.txt` - Should show crawl delay of 2 +- `http://localhost:4321/llms.txt` - Should show your description and key features +- `http://localhost:4321/humans.txt` - Should show your team info and tech stack + +## What You've Learned + +You now know how to: +- Add site descriptions for AI assistants +- Credit your team in humans.txt +- Document your tech stack +- List key features for AI discovery +- Adjust bot crawl behavior + +## Your Complete Configuration + +Here's what you've built: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + crawlDelay: 2, + }, + llms: { + description: 'A personal blog about web development, focusing on modern JavaScript frameworks and best practices', + keyFeatures: [ + 'In-depth tutorials on modern web development', + 'Code examples with live demos', + 'Weekly newsletter with web dev tips', + 'Open source project showcase', + ], + }, + humans: { + team: [ + { + name: 'Your Name', + role: 'Developer', + contact: 'you@example.com', + location: 'Your City', + } + ], + site: { + lastUpdate: 'auto', + language: 'English', + doctype: 'HTML5', + techStack: ['Astro', 'TypeScript', 'React', 'Tailwind CSS'], + } + } + }) + ] +}); +``` + +## Next Steps + +Now that you have the basics configured, explore more advanced features: + +- [Configure robots.txt](/tutorials/configure-robots/) - Control which bots can access what +- [Setup llms.txt](/tutorials/setup-llms/) - Provide detailed instructions for AI +- [Create humans.txt](/tutorials/create-humans/) - Add story and fun facts +- [Security & Canary](/tutorials/security-canary/) - Add security contact info + +## Troubleshooting + +### Configuration Not Taking Effect? + +Make sure to rebuild after changing config: + +```bash +npm run build +``` + +### Type Errors in Config? + +Install TypeScript definitions if needed: + +```bash +npm install -D @types/node +``` + +### Need More Options? + +Check the [Configuration Reference](/reference/configuration/) for all available options. diff --git a/docs/src/content/docs/tutorials/configure-robots.md b/docs/src/content/docs/tutorials/configure-robots.md index e598fd4..e3a379e 100644 --- a/docs/src/content/docs/tutorials/configure-robots.md +++ b/docs/src/content/docs/tutorials/configure-robots.md @@ -3,29 +3,396 @@ title: Configure robots.txt description: Customize your robots.txt file --- -Learn how to configure robots.txt to control search engine and bot crawling behavior. +In this tutorial, you'll learn how to configure robots.txt to control which bots can access your site and how they should behave. -:::note[Work in Progress] -This page is currently being developed. Check back soon for complete documentation. -::: +## What You'll Build -## Coming Soon +By the end of this tutorial, you'll have: +- Custom bot access rules +- Specific rules for different user agents +- Protected admin and private pages +- Configured LLM bot access -This section will include: -- Detailed explanations -- Code examples -- Best practices -- Common patterns -- Troubleshooting tips +## Before You Start -## Related Pages +Make sure you have: +- Completed the [Basic Setup](/tutorials/basic-setup/) tutorial +- An understanding of what robots.txt does +- Pages you want to protect from bots -- [Configuration Reference](/reference/configuration/) -- [API Reference](/reference/api/) -- [Examples](/examples/ecommerce/) +## Step 1: Understanding the Default robots.txt -## Need Help? +Build your site and look at the default robots.txt: -- Check our [FAQ](/community/faq/) -- Visit [Troubleshooting](/community/troubleshooting/) -- Open an issue on [GitHub](https://github.com/withastro/astro-discovery/issues) +```bash +npm run build +cat dist/robots.txt +``` + +You'll see: + +```txt +User-agent: * +Allow: / + +# Sitemaps +Sitemap: https://your-site.com/sitemap-index.xml + +# LLM-specific resources +User-agent: Anthropic-AI +User-agent: Claude-Web +User-agent: GPTBot +User-agent: ChatGPT-User +User-agent: cohere-ai +User-agent: Google-Extended +Allow: /llms.txt + +Crawl-delay: 1 +``` + +This allows all bots full access to your site. Let's customize it! + +## Step 2: Block Private Pages + +Let's say you have admin and draft pages you want to protect. Update your config: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + additionalAgents: [ + { + userAgent: '*', + disallow: ['/admin', '/draft', '/private'], + } + ], + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/robots.txt +``` + +You'll now see: + +```txt +User-agent: * +Disallow: /admin +Disallow: /draft +Disallow: /private +Allow: / +``` + +All bots are now blocked from those paths! + +## Step 3: Allow Specific Paths Only + +Maybe you want to limit a specific bot to only certain paths. Let's allow a custom bot to access only the API: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + additionalAgents: [ + { + userAgent: '*', + disallow: ['/admin', '/draft', '/private'], + }, + { + userAgent: 'MyCustomBot', + allow: ['/api'], + disallow: ['/'], + } + ], + } + }) + ] +}); +``` + +Build and verify: + +```bash +npm run build +cat dist/robots.txt +``` + +Now you'll see rules for MyCustomBot: + +```txt +User-agent: MyCustomBot +Allow: /api +Disallow: / +``` + +This bot can only access `/api` paths! + +## Step 4: Block a Troublesome Bot Completely + +Let's block a specific bot entirely: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + additionalAgents: [ + { + userAgent: '*', + disallow: ['/admin', '/draft', '/private'], + }, + { + userAgent: 'BadBot', + disallow: ['/'], + } + ], + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/robots.txt +``` + +BadBot is now completely blocked! + +## Step 5: Adjust Crawl Delay + +If your server is getting hammered by bots, increase the crawl delay: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + crawlDelay: 5, // Wait 5 seconds between requests + additionalAgents: [ + { + userAgent: '*', + disallow: ['/admin', '/draft', '/private'], + } + ], + } + }) + ] +}); +``` + +Build and verify: + +```bash +npm run build +cat dist/robots.txt +``` + +You'll see `Crawl-delay: 5` in the file. + +## Step 6: Customize LLM Bot Access + +By default, LLM bots can access everything. Let's control this: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + llmBots: { + enabled: true, + agents: ['Anthropic-AI', 'Claude-Web'], // Only allow these AI bots + }, + additionalAgents: [ + { + userAgent: '*', + disallow: ['/admin', '/draft', '/private'], + }, + { + userAgent: 'GPTBot', // Block OpenAI's bot specifically + disallow: ['/'], + } + ], + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/robots.txt +``` + +Now only Anthropic AI bots have special access! + +## Step 7: Disable LLM Bots Entirely + +If you don't want AI bots crawling your site: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + robots: { + llmBots: { + enabled: false, // No special LLM bot rules + }, + } + }) + ] +}); +``` + +Build and verify - the LLM-specific section will be gone! + +## Step 8: Test Your Configuration + +Start the dev server: + +```bash +npm run dev +``` + +Visit `http://localhost:4321/robots.txt` to see your rules in action. + +## What You've Learned + +You now know how to: +- Block specific paths from all bots +- Create bot-specific access rules +- Completely block troublesome bots +- Adjust crawl delay to protect your server +- Control LLM bot access +- Enable or disable AI bot crawling + +## Common Patterns + +### E-commerce Site + +```typescript +discovery({ + robots: { + crawlDelay: 2, + additionalAgents: [ + { + userAgent: '*', + disallow: ['/checkout', '/account', '/admin'], + }, + { + userAgent: 'PriceScraperBot', + disallow: ['/'], + } + ], + } +}) +``` + +### Blog with Drafts + +```typescript +discovery({ + robots: { + additionalAgents: [ + { + userAgent: '*', + disallow: ['/draft', '/preview'], + } + ], + } +}) +``` + +### API Platform + +```typescript +discovery({ + robots: { + llmBots: { + enabled: true, // Let AI bots learn from docs + }, + additionalAgents: [ + { + userAgent: '*', + allow: ['/docs', '/api/v1'], + disallow: ['/admin', '/api/internal'], + } + ], + } +}) +``` + +## Testing Your Rules + +After configuring, always test: + +1. **Build and inspect:** + ```bash + npm run build + cat dist/robots.txt + ``` + +2. **Verify in dev:** + ```bash + npm run dev + # Visit http://localhost:4321/robots.txt + ``` + +3. **Test with Google's tool:** + Use Google Search Console's robots.txt tester after deploying + +## Next Steps + +- [Setup llms.txt](/tutorials/setup-llms/) - Configure AI assistant instructions +- [Create humans.txt](/tutorials/create-humans/) - Add team credits +- [Block Bots How-To](/how-to/block-bots/) - Advanced bot blocking patterns + +## Troubleshooting + +### Bots Still Accessing Blocked Pages? + +Remember that robots.txt is a suggestion, not enforcement. Badly behaved bots may ignore it. Consider: +- Server-level blocking with .htaccess or nginx rules +- Rate limiting +- IP blocking + +### Rules Not Taking Effect? + +Make sure to: +1. Rebuild after config changes +2. Clear CDN cache if using one +3. Wait for bots to re-crawl robots.txt +4. Check that robots.txt is at the root URL + +### LLM Bots Not Listed? + +The default LLM bots are: +- Anthropic-AI +- Claude-Web +- GPTBot +- ChatGPT-User +- cohere-ai +- Google-Extended + +New bots appear regularly - add them manually to the agents array. diff --git a/docs/src/content/docs/tutorials/create-humans.md b/docs/src/content/docs/tutorials/create-humans.md index b7f6ae3..4b6bb21 100644 --- a/docs/src/content/docs/tutorials/create-humans.md +++ b/docs/src/content/docs/tutorials/create-humans.md @@ -3,29 +3,499 @@ title: Create humans.txt description: Add team credits and tech stack information --- -Learn how to create a humans.txt file to credit your team and document your tech stack. +In this tutorial, you'll learn how to create a humans.txt file to credit your team, tell your project's story, and document your technology stack. -:::note[Work in Progress] -This page is currently being developed. Check back soon for complete documentation. -::: +## What You'll Build -## Coming Soon +By the end of this tutorial, you'll have: +- Team member credits with contact information +- A list of acknowledgments +- Documented tech stack +- Your project's story +- Fun facts about your project -This section will include: -- Detailed explanations -- Code examples -- Best practices -- Common patterns -- Troubleshooting tips +## Before You Start -## Related Pages +Make sure you have: +- Completed the [Basic Setup](/tutorials/basic-setup/) tutorial +- Information about your team members +- A sense of your project's story -- [Configuration Reference](/reference/configuration/) -- [API Reference](/reference/api/) -- [Examples](/examples/ecommerce/) +## Step 1: Add Team Members -## Need Help? +Open your `astro.config.mjs` and add your team: -- Check our [FAQ](/community/faq/) -- Visit [Troubleshooting](/community/troubleshooting/) -- Open an issue on [GitHub](https://github.com/withastro/astro-discovery/issues) +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [ + { + name: 'Jane Developer', + role: 'Lead Developer', + contact: 'jane@example.com', + location: 'San Francisco, CA', + } + ], + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/humans.txt +``` + +You'll see: + +```txt +/* TEAM */ + +Name: Jane Developer +Role: Lead Developer +Contact: jane@example.com +Location: San Francisco, CA +``` + +Your first team member is credited! + +## Step 2: Add Multiple Team Members + +Let's add more people: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [ + { + name: 'Jane Developer', + role: 'Lead Developer', + contact: 'jane@example.com', + location: 'San Francisco, CA', + twitter: '@janedev', + github: 'janedev', + }, + { + name: 'Bob Designer', + role: 'UI/UX Designer', + contact: 'bob@example.com', + location: 'Austin, TX', + twitter: '@bobdesigns', + }, + { + name: 'Alice DevOps', + role: 'Infrastructure Engineer', + location: 'Remote', + github: 'alice-ops', + }, + ], + } + }) + ] +}); +``` + +Build and verify - all team members are now listed! + +## Step 3: Add Thanks and Acknowledgments + +Credit the people and projects that helped: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [ + { + name: 'Jane Developer', + role: 'Lead Developer', + contact: 'jane@example.com', + } + ], + thanks: [ + 'The amazing Astro team', + 'Our supportive open source community', + 'Coffee, for making this possible', + 'All our beta testers', + 'Stack Overflow (you know why)', + ], + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/humans.txt +``` + +You'll see a new THANKS section! + +## Step 4: Document Your Tech Stack + +Let's tell people what you built with: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [/* ... */], + thanks: [/* ... */], + site: { + lastUpdate: 'auto', // Automatically uses current date + language: 'English', + doctype: 'HTML5', + ide: 'VS Code', + techStack: [ + 'Astro', + 'TypeScript', + 'React', + 'Tailwind CSS', + 'PostgreSQL', + ], + standards: [ + 'HTML5', + 'CSS3', + 'WCAG 2.1 AA', + ], + components: [ + 'React', + 'Astro Components', + ], + software: [ + 'Figma', + 'Docker', + 'GitHub Actions', + ], + }, + } + }) + ] +}); +``` + +Build and verify - comprehensive tech documentation! + +## Step 5: Tell Your Story + +Add a narrative about your project: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [/* ... */], + thanks: [/* ... */], + site: {/* ... */}, + story: ` +This project started in early 2024 when we realized there was no simple +way to build fast, modern websites without complex tooling. We fell in +love with Astro and built this site to showcase what's possible. + +Three months, 47 cups of coffee, and countless late nights later, we +launched. The response from the community has been incredible, and we're +just getting started. + `.trim(), + } + }) + ] +}); +``` + +Build and check - your story is now part of humans.txt! + +## Step 6: Add Fun Facts + +Make it personal with fun facts: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [/* ... */], + thanks: [/* ... */], + site: {/* ... */}, + story: `...`, + funFacts: [ + 'Built entirely on mechanical keyboards', + 'Fueled by 347 cups of coffee and 128 energy drinks', + 'First deployed from a coffee shop in Portland', + 'Named after a joke that nobody remembers anymore', + 'Our mascot is a rubber duck named Herbert', + ], + } + }) + ] +}); +``` + +Build and verify - personality added! + +## Step 7: Add Your Philosophy + +Share your project values: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [/* ... */], + philosophy: [ + 'Users first, always', + 'Fast is a feature', + 'Accessibility is not optional', + 'Simple over complex', + 'Open source by default', + ], + } + }) + ] +}); +``` + +Build and check - your values are documented! + +## Step 8: Add Custom Sections + +Need something specific? Add custom sections: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + humans: { + team: [/* ... */], + customSections: { + 'SUSTAINABILITY': 'Hosted on 100% renewable energy, carbon-offset delivery', + 'COMMUNITY': 'Join us on Discord: discord.gg/yourserver', + 'HIRING': 'We\'re hiring! Check careers.example.com', + }, + } + }) + ] +}); +``` + +## What You've Learned + +You now know how to: +- Credit team members with full details +- Add social links (Twitter, GitHub) +- Thank contributors and inspirations +- Document your complete tech stack +- Tell your project's story +- Add fun facts and personality +- Share your project philosophy +- Create custom sections + +## Complete Example + +Here's a full, real-world humans.txt configuration: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://awesome-project.com', + integrations: [ + discovery({ + humans: { + team: [ + { + name: 'Sarah Chen', + role: 'Founder & Lead Developer', + contact: 'sarah@awesome-project.com', + location: 'Seattle, WA', + twitter: '@sarahchen', + github: 'sarahchen', + }, + { + name: 'Marcus Johnson', + role: 'Senior Developer', + contact: 'marcus@awesome-project.com', + location: 'Austin, TX', + github: 'marcusj', + }, + { + name: 'Yuki Tanaka', + role: 'Designer & UX Lead', + location: 'Tokyo, Japan', + twitter: '@yukidesigns', + }, + ], + + thanks: [ + 'The Astro core team for building an amazing framework', + 'Our 1,247 GitHub stargazers', + 'Beta testers who found all the edge cases', + 'The open source community', + 'Our families for putting up with late-night deploys', + ], + + site: { + lastUpdate: 'auto', + language: 'English / 日本語', + doctype: 'HTML5', + ide: 'VS Code + Vim', + techStack: [ + 'Astro 4.0', + 'TypeScript', + 'React', + 'Tailwind CSS', + 'PostgreSQL', + 'Redis', + ], + standards: [ + 'HTML5', + 'CSS3', + 'WCAG 2.1 AA', + 'JSON:API', + ], + components: [ + 'React', + 'Astro Islands', + 'Headless UI', + ], + software: [ + 'Figma', + 'Docker', + 'GitHub Actions', + 'Playwright', + ], + }, + + story: ` +Awesome Project was born from a simple frustration: building modern +web apps was too complicated. We wanted something fast, simple, and +delightful to use. + +In January 2024, Sarah had the idea over coffee. By March, we had a +working prototype. In June, we launched to the world. The response +was overwhelming - thousands of developers joined our community in +the first week. + +Today, Awesome Project powers over 10,000 websites worldwide, from +personal blogs to enterprise applications. But we're just getting +started. + `.trim(), + + funFacts: [ + 'Written entirely in coffee shops across 3 continents', + 'Our first commit was made on a plane at 30,000 feet', + 'The codebase includes exactly 42 easter eggs', + 'Marcus has never used a mouse - keyboard shortcuts only', + 'Yuki designed the logo in 7 minutes on a napkin', + 'We\'ve gone through 23 different logo iterations', + 'The project mascot is a caffeinated squirrel', + ], + + philosophy: [ + 'Users come first, always', + 'Performance is a feature, not an afterthought', + 'Accessibility is mandatory, not optional', + 'Simple solutions beat complex ones', + 'Open source by default', + 'Documentation is just as important as code', + 'Be kind, be curious, be helpful', + ], + } + }) + ] +}); +``` + +This generates a comprehensive humans.txt file! + +## Testing Your humans.txt + +Build and review: + +```bash +npm run build +cat dist/humans.txt +``` + +Or in dev mode: + +```bash +npm run dev +# Visit http://localhost:4321/humans.txt +``` + +## Tips for Great Credits + +### Be Genuine + +Don't add fake team members or exaggerated thanks. People appreciate authenticity. + +### Update Regularly + +When team members change or the tech stack evolves, update humans.txt. + +### Keep It Fun + +humans.txt is one place where personality is encouraged. Add jokes, fun facts, and quirks! + +### Credit Everyone + +Don't forget contractors, beta testers, and community contributors. + +### Link Appropriately + +Use Twitter and GitHub handles so people can connect with your team. + +## Next Steps + +- [Security & Canary](/tutorials/security-canary/) - Add security contact info +- [WebFinger](/tutorials/webfinger/) - Enable federated discovery +- [Add Team Members How-To](/how-to/add-team-members/) - Advanced team management + +## Troubleshooting + +### Auto Date Not Working? + +Make sure you're using `'auto'` as a string: + +```typescript +lastUpdate: 'auto' // Correct +lastUpdate: auto // Wrong +``` + +### Too Much Information? + +humans.txt can be as short or long as you want. Include only what feels right for your project. + +### Character Encoding Issues? + +Make sure your config file is saved as UTF-8, especially if using non-ASCII characters. + +### Social Links Not Showing? + +Optional fields only appear if you provide them. It's fine to omit twitter or github if not applicable. diff --git a/docs/src/content/docs/tutorials/security-canary.md b/docs/src/content/docs/tutorials/security-canary.md index a0d3b76..4f680e6 100644 --- a/docs/src/content/docs/tutorials/security-canary.md +++ b/docs/src/content/docs/tutorials/security-canary.md @@ -3,29 +3,519 @@ title: Security & Canary Files description: Set up security.txt and canary.txt --- -Configure security contact information and warrant canaries for transparency. +In this tutorial, you'll learn how to set up security.txt for responsible disclosure and canary.txt for transparency about government requests. -:::note[Work in Progress] -This page is currently being developed. Check back soon for complete documentation. -::: +## What You'll Build -## Coming Soon +By the end of this tutorial, you'll have: +- RFC 9116 compliant security.txt +- Contact information for security researchers +- A warrant canary for transparency +- Automated expiration dates +- PGP encryption details (optional) -This section will include: -- Detailed explanations -- Code examples -- Best practices -- Common patterns -- Troubleshooting tips +## Before You Start -## Related Pages +Make sure you have: +- Completed the [Basic Setup](/tutorials/basic-setup/) tutorial +- A security contact email or URL +- Understanding of responsible disclosure -- [Configuration Reference](/reference/configuration/) -- [API Reference](/reference/api/) -- [Examples](/examples/ecommerce/) +## Part 1: Setting Up security.txt -## Need Help? +### Step 1: Add Basic Security Contact -- Check our [FAQ](/community/faq/) -- Visit [Troubleshooting](/community/troubleshooting/) -- Open an issue on [GitHub](https://github.com/withastro/astro-discovery/issues) +Open your `astro.config.mjs` and add security configuration: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + security: { + contact: 'security@your-site.com', + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/.well-known/security.txt +``` + +You'll see: + +```txt +Contact: mailto:security@your-site.com +Expires: 2026-01-08T00:00:00.000Z +Canonical: https://your-site.com/.well-known/security.txt +``` + +Your site now has an RFC 9116 compliant security.txt! + +### Step 2: Add Expiration Date + +The integration auto-generates expiration (1 year), but you can customize it: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + security: { + contact: 'security@your-site.com', + expires: '2025-12-31T23:59:59Z', // Custom expiration + } + }) + ] +}); +``` + +Build and verify the custom expiration! + +### Step 3: Add PGP Encryption Key + +Provide your PGP key for encrypted communications: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + security: { + contact: 'security@your-site.com', + expires: 'auto', // Use auto-generation + encryption: 'https://your-site.com/pgp-key.txt', + } + }) + ] +}); +``` + +Build and check - encryption URL is now included! + +### Step 4: Add Acknowledgments Page + +Give credit to security researchers: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + security: { + contact: 'security@your-site.com', + expires: 'auto', + encryption: 'https://your-site.com/pgp-key.txt', + acknowledgments: 'https://your-site.com/security/hall-of-fame', + } + }) + ] +}); +``` + +Build and verify! + +### Step 5: Add Security Policy + +Link to your responsible disclosure policy: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + security: { + contact: 'security@your-site.com', + expires: 'auto', + encryption: 'https://your-site.com/pgp-key.txt', + acknowledgments: 'https://your-site.com/security/hall-of-fame', + policy: 'https://your-site.com/security/policy', + preferredLanguages: ['en', 'es'], + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/.well-known/security.txt +``` + +Complete security.txt is now ready! + +## Part 2: Setting Up canary.txt + +### Step 1: Add Basic Canary + +A warrant canary signals you haven't received secret government requests: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + security: { + contact: 'security@your-site.com', + }, + canary: { + organization: 'Your Company Inc', + contact: 'canary@your-site.com', + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/.well-known/canary.txt +``` + +You'll see: + +```txt +-----BEGIN CANARY STATEMENT----- +Organization: Your Company Inc +Contact: canary@your-site.com +Issued: 2025-01-08T12:00:00.000Z +Expires: 2025-02-12T12:00:00.000Z + +As of the date above, Your Company Inc has not received any national +security orders or gag orders. +-----END CANARY STATEMENT----- +``` + +### Step 2: Set Update Frequency + +Control how often you update the canary: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + canary: { + organization: 'Your Company Inc', + contact: 'canary@your-site.com', + frequency: 'monthly', // daily, weekly, monthly, quarterly, yearly + } + }) + ] +}); +``` + +The expiration auto-adjusts: +- daily: 2 days +- weekly: 10 days +- monthly: 35 days (default) +- quarterly: 100 days +- yearly: 380 days + +### Step 3: Add Specific Statements + +Declare what you haven't received: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + canary: { + organization: 'Your Company Inc', + contact: 'canary@your-site.com', + frequency: 'monthly', + statements: [ + { + type: 'nsl', + description: 'National Security Letters', + received: false, + }, + { + type: 'fisa', + description: 'FISA court orders', + received: false, + }, + { + type: 'gag', + description: 'Gag orders preventing disclosure', + received: false, + }, + { + type: 'subpoena', + description: 'Government subpoenas for user data', + received: false, + }, + ], + } + }) + ] +}); +``` + +Build and verify - specific statements are listed! + +### Step 4: Add Personnel Statement + +Confirm no team members are under duress: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + canary: { + organization: 'Your Company Inc', + contact: 'canary@your-site.com', + frequency: 'monthly', + statements: [ + { + type: 'nsl', + description: 'National Security Letters', + received: false, + }, + ], + personnelStatement: true, // Add duress check + } + }) + ] +}); +``` + +Build and check - personnel confirmation is added! + +### Step 5: Add PGP Verification + +Sign your canary with PGP: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + canary: { + organization: 'Your Company Inc', + contact: 'canary@your-site.com', + frequency: 'monthly', + statements: [/* ... */], + verification: 'PGP Signature: https://your-site.com/canary.txt.asc', + } + }) + ] +}); +``` + +Build and verify! + +### Step 6: Add Blockchain Proof (Advanced) + +For maximum transparency, add blockchain verification: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + canary: { + organization: 'Your Company Inc', + contact: 'canary@your-site.com', + frequency: 'monthly', + blockchainProof: { + network: 'Bitcoin', + address: '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa', + txHash: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', + timestamp: '2025-01-08T12:00:00Z', + }, + } + }) + ] +}); +``` + +This proves the canary was published at a specific time! + +## Step 7: Test Both Files + +Start dev server: + +```bash +npm run dev +``` + +Visit: +- `http://localhost:4321/.well-known/security.txt` +- `http://localhost:4321/.well-known/canary.txt` + +Both files should be accessible! + +## What You've Learned + +You now know how to: +- Create RFC 9116 compliant security.txt +- Add security contact and encryption details +- Link to security policies and acknowledgments +- Set up a warrant canary +- Configure update frequency +- Add specific statements +- Verify with PGP or blockchain + +## Complete Example + +Here's a comprehensive security and canary setup: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://example.com', + integrations: [ + discovery({ + security: { + contact: 'security@example.com', + expires: 'auto', // 1 year from now + encryption: 'https://example.com/pgp-key.txt', + acknowledgments: 'https://example.com/security/hall-of-fame', + preferredLanguages: ['en', 'es'], + policy: 'https://example.com/security/policy', + hiring: 'https://example.com/security/jobs', + }, + + canary: { + organization: 'Example Corp', + contact: 'canary@example.com', + frequency: 'monthly', + + statements: [ + { + type: 'nsl', + description: 'National Security Letters', + received: false, + }, + { + type: 'fisa', + description: 'FISA court orders', + received: false, + }, + { + type: 'gag', + description: 'Gag orders', + received: false, + }, + { + type: 'warrant', + description: 'Government search warrants', + received: false, + }, + ], + + additionalStatement: 'We are committed to transparency and protecting user privacy.', + personnelStatement: true, + verification: 'PGP Signature: https://example.com/canary.txt.asc', + + blockchainProof: { + network: 'Bitcoin', + address: '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa', + }, + } + }) + ] +}); +``` + +## Best Practices + +### Security.txt + +1. **Keep it updated**: Set calendar reminders before expiration +2. **Test your contact**: Make sure security@example.com works +3. **Provide encryption**: Security researchers prefer encrypted communication +4. **Be responsive**: Respond to reports within 24-48 hours +5. **Give credit**: Maintain a hall of fame for responsible disclosers + +### Canary.txt + +1. **Update regularly**: Stick to your frequency schedule +2. **Automate**: Set up automated deployment/updates +3. **Sign it**: Use PGP signatures for verification +4. **Be consistent**: Always update on the same day +5. **Archive old canaries**: Keep a history for transparency +6. **Don't lie**: Only use if you can commit to honesty + +## Important Legal Notes + +### Warrant Canaries + +Consult with legal counsel before implementing a warrant canary: +- Laws vary by jurisdiction +- May not be legally effective everywhere +- Could have unintended consequences +- Should be part of broader transparency efforts + +### Security.txt + +- Must be accurate and up-to-date +- Contact must actually work +- Expiration date is required by RFC 9116 +- Should be part of a real security program + +## Troubleshooting + +### security.txt Not in .well-known? + +The file should be at `/.well-known/security.txt` per RFC 9116. Check: + +```bash +ls dist/.well-known/ +``` + +### Canary Expired? + +Rebuild your site regularly to update timestamps: + +```bash +npm run build +``` + +Consider automating rebuilds based on your frequency. + +### Email Not Showing mailto: Prefix? + +The integration auto-adds it. Just provide: + +```typescript +contact: 'security@example.com' // Becomes mailto:security@example.com +``` + +### Want Multiple Contacts? + +Use an array: + +```typescript +contact: ['security@example.com', 'https://example.com/security/report'] +``` + +## Next Steps + +- [WebFinger Discovery](/tutorials/webfinger/) - Enable federated discovery +- [Environment Config](/how-to/environment-config/) - Different configs per environment +- [Security Explained](/explanation/security-explained/) - Deep dive into security.txt +- [Canary Explained](/explanation/canary-explained/) - Understanding warrant canaries + +## Additional Resources + +- [RFC 9116 - security.txt](https://www.rfc-editor.org/rfc/rfc9116.html) +- [securitytxt.org](https://securitytxt.org/) +- [Warrant Canary FAQ](https://www.eff.org/deeplinks/2014/04/warrant-canary-faq) diff --git a/docs/src/content/docs/tutorials/setup-llms.md b/docs/src/content/docs/tutorials/setup-llms.md index 2a0cdef..f6050f4 100644 --- a/docs/src/content/docs/tutorials/setup-llms.md +++ b/docs/src/content/docs/tutorials/setup-llms.md @@ -3,29 +3,436 @@ title: Setup llms.txt description: Configure AI assistant discovery and instructions --- -Set up llms.txt to help AI assistants understand and interact with your site. +In this tutorial, you'll learn how to set up llms.txt to help AI assistants like Claude, ChatGPT, and others understand and interact with your site effectively. -:::note[Work in Progress] -This page is currently being developed. Check back soon for complete documentation. -::: +## What You'll Build -## Coming Soon +By the end of this tutorial, you'll have: +- A comprehensive site description for AI assistants +- Specific instructions for how AI should help users +- Documented API endpoints +- Listed important pages and features +- Defined your brand voice -This section will include: -- Detailed explanations -- Code examples -- Best practices -- Common patterns -- Troubleshooting tips +## Before You Start -## Related Pages +Make sure you have: +- Completed the [Basic Setup](/tutorials/basic-setup/) tutorial +- A clear understanding of your site's purpose +- Knowledge of what AI assistants should know about your site -- [Configuration Reference](/reference/configuration/) -- [API Reference](/reference/api/) -- [Examples](/examples/ecommerce/) +## Step 1: Start with a Good Description -## Need Help? +Open your `astro.config.mjs` and add a clear, concise description: -- Check our [FAQ](/community/faq/) -- Visit [Troubleshooting](/community/troubleshooting/) -- Open an issue on [GitHub](https://github.com/withastro/astro-discovery/issues) +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development, featuring tutorials, code examples, and best practices for building fast, accessible websites with Astro', + } + }) + ] +}); +``` + +Build and check: + +```bash +npm run build +cat dist/llms.txt +``` + +You'll see your description prominently displayed! + +## Step 2: List Key Features + +Help AI assistants understand what makes your site special: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development, featuring tutorials, code examples, and best practices for building fast, accessible websites with Astro', + keyFeatures: [ + 'Step-by-step tutorials for beginners to advanced developers', + 'Interactive code examples with live previews', + 'Performance optimization guides', + 'Accessibility best practices', + 'Weekly newsletter with web dev tips', + ], + } + }) + ] +}); +``` + +Build and verify: + +```bash +npm run build +cat dist/llms.txt +``` + +Your key features are now listed! + +## Step 3: Highlight Important Pages + +Tell AI assistants about your most valuable content: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development', + keyFeatures: [ + 'Step-by-step tutorials', + 'Interactive code examples', + ], + importantPages: [ + { + name: 'Getting Started Guide', + path: '/getting-started', + description: 'Begin your web development journey', + }, + { + name: 'Tutorial Library', + path: '/tutorials', + description: 'Comprehensive tutorials for all skill levels', + }, + { + name: 'API Documentation', + path: '/api', + description: 'Complete API reference', + }, + ], + } + }) + ] +}); +``` + +Build and check - AI assistants can now find your key pages! + +## Step 4: Provide Specific Instructions + +This is where you really help AI assistants. Give them clear guidance: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development', + instructions: ` +When helping users with this site: + +1. Start by checking the Getting Started guide for new users +2. Reference specific tutorials when answering technical questions +3. Link to the API documentation for detailed method references +4. Encourage users to try the interactive code examples +5. Suggest subscribing to the newsletter for ongoing learning +6. Always provide working code examples when possible +7. Mention performance and accessibility considerations + `.trim(), + } + }) + ] +}); +``` + +Build and verify: + +```bash +npm run build +cat dist/llms.txt +``` + +Now AI assistants have clear instructions! + +## Step 5: Document API Endpoints + +If your site has an API, document it: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development', + instructions: `When helping users...`, + apiEndpoints: [ + { + path: '/api/tutorials', + method: 'GET', + description: 'List all available tutorials with filtering options', + }, + { + path: '/api/search', + method: 'GET', + description: 'Search site content by keyword', + }, + { + path: '/api/subscribe', + method: 'POST', + description: 'Subscribe to the newsletter', + }, + ], + } + }) + ] +}); +``` + +Build and check - your API is now documented! + +## Step 6: Define Your Tech Stack + +Help AI assistants understand your technical foundation: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development', + techStack: { + frontend: ['Astro', 'React', 'TypeScript', 'Tailwind CSS'], + backend: ['Node.js', 'Express'], + ai: ['OpenAI API', 'Claude API'], + other: ['PostgreSQL', 'Redis', 'Docker'], + }, + } + }) + ] +}); +``` + +Build and verify - AI knows your stack! + +## Step 7: Set Your Brand Voice + +Guide AI assistants on how to communicate about your site: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development', + brandVoice: [ + 'Friendly and approachable, never condescending', + 'Technical but accessible - explain complex topics clearly', + 'Encouraging and supportive of learning', + 'Practical and example-driven', + 'Honest about limitations and edge cases', + ], + } + }) + ] +}); +``` + +Build and check the results! + +## Step 8: Add Custom Sections + +Need something specific? Add custom sections: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + llms: { + description: 'A comprehensive guide to modern web development', + customSections: { + 'Community Guidelines': 'Be respectful, help others, share knowledge', + 'Support': 'For questions, visit our Discord or open a GitHub issue', + 'Contributing': 'We welcome contributions! See CONTRIBUTING.md', + }, + } + }) + ] +}); +``` + +## Step 9: Test with a Dev Server + +Start your dev server and check the results: + +```bash +npm run dev +``` + +Visit `http://localhost:4321/llms.txt` and review everything. + +## What You've Learned + +You now know how to: +- Write effective site descriptions for AI +- List key features and important pages +- Provide specific instructions to AI assistants +- Document API endpoints +- Define your tech stack +- Set your brand voice +- Add custom sections + +## Complete Example + +Here's a full, real-world example: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://webdev-academy.com', + integrations: [ + discovery({ + llms: { + description: 'WebDev Academy is an interactive learning platform for modern web development, offering hands-on tutorials, real-world projects, and a supportive community', + + keyFeatures: [ + 'Interactive coding challenges with instant feedback', + 'Project-based learning with real-world applications', + 'Peer code review and mentorship program', + 'Career guidance and interview preparation', + 'Regularly updated with latest web technologies', + ], + + importantPages: [ + { + name: 'Learning Paths', + path: '/paths', + description: 'Structured curricula for different skill levels and goals', + }, + { + name: 'Interactive Challenges', + path: '/challenges', + description: 'Hands-on coding exercises with progressive difficulty', + }, + { + name: 'Community Forum', + path: '/community', + description: 'Ask questions, share projects, get feedback', + }, + ], + + instructions: ` +When helping users with WebDev Academy: + +1. Assess their skill level first - we have content for beginners to advanced +2. Recommend appropriate learning paths based on their goals +3. Encourage hands-on practice with our interactive challenges +4. Suggest joining the community forum for peer support +5. Link to relevant tutorials and documentation +6. Provide code examples that users can test in our playground +7. Emphasize learning by building real projects +8. Be patient and encouraging - everyone starts somewhere + `.trim(), + + apiEndpoints: [ + { + path: '/api/challenges', + method: 'GET', + description: 'List coding challenges by difficulty and topic', + }, + { + path: '/api/progress', + method: 'GET', + description: 'Get user learning progress and achievements', + }, + { + path: '/api/submit', + method: 'POST', + description: 'Submit challenge solutions for automated testing', + }, + ], + + techStack: { + frontend: ['Astro', 'React', 'TypeScript', 'Tailwind CSS'], + backend: ['Node.js', 'Fastify', 'PostgreSQL'], + ai: ['Claude API for code review feedback'], + other: ['Docker', 'Redis', 'Playwright for testing'], + }, + + brandVoice: [ + 'Encouraging and supportive - learning to code is hard!', + 'Clear and jargon-free explanations', + 'Practical and project-focused', + 'Honest about the learning curve', + 'Community-oriented and collaborative', + ], + } + }) + ] +}); +``` + +## Next Steps + +- [Create humans.txt](/tutorials/create-humans/) - Add team credits +- [Security & Canary](/tutorials/security-canary/) - Add security info +- [Customize LLM Instructions](/how-to/customize-llm-instructions/) - Advanced instruction patterns + +## Tips for Great AI Instructions + +### Be Specific + +Bad: "Help users with the site" +Good: "Search the tutorial library first, then provide step-by-step guidance with code examples" + +### Give Context + +Bad: "We have docs" +Good: "Documentation is at /docs with beginner, intermediate, and advanced sections" + +### Set Expectations + +Bad: "Answer questions" +Good: "If the question is beyond the site's scope, acknowledge it and suggest external resources" + +### Update Regularly + +As your site grows, keep instructions current: +- Add new features to keyFeatures +- Update important pages +- Revise instructions based on common user questions + +## Troubleshooting + +### Instructions Too Long? + +Keep it concise - AI assistants have token limits. Focus on: +1. Most common user needs +2. Most important pages +3. Key navigation patterns + +### Not Seeing Changes? + +Remember to rebuild: + +```bash +npm run build +``` + +### Want to Test AI Understanding? + +Ask an AI assistant like Claude: +"What can you tell me about [your-site.com]?" + +The assistant should reference your llms.txt! diff --git a/docs/src/content/docs/tutorials/webfinger.md b/docs/src/content/docs/tutorials/webfinger.md index da67246..115d964 100644 --- a/docs/src/content/docs/tutorials/webfinger.md +++ b/docs/src/content/docs/tutorials/webfinger.md @@ -3,29 +3,538 @@ title: WebFinger Discovery description: Enable WebFinger resource discovery --- -Set up WebFinger for ActivityPub, OpenID Connect, and other federated protocols. +In this tutorial, you'll learn how to set up WebFinger for federated discovery, enabling ActivityPub (Mastodon), OpenID Connect, and other federated protocols. -:::note[Work in Progress] -This page is currently being developed. Check back soon for complete documentation. -::: +## What You'll Build -## Coming Soon +By the end of this tutorial, you'll have: +- WebFinger endpoint at `/.well-known/webfinger` +- Resource discovery for team members +- ActivityPub/Mastodon integration +- OpenID Connect support (optional) +- Dynamic resource lookups -This section will include: -- Detailed explanations -- Code examples -- Best practices -- Common patterns -- Troubleshooting tips +## Before You Start -## Related Pages +Make sure you have: +- Completed the [Basic Setup](/tutorials/basic-setup/) tutorial +- Understanding of what WebFinger is used for +- Knowledge of the resources you want to make discoverable -- [Configuration Reference](/reference/configuration/) -- [API Reference](/reference/api/) -- [Examples](/examples/ecommerce/) +## What is WebFinger? -## Need Help? +WebFinger (RFC 7033) lets people and services discover information about resources using simple identifiers like email addresses or usernames. It powers: -- Check our [FAQ](/community/faq/) -- Visit [Troubleshooting](/community/troubleshooting/) -- Open an issue on [GitHub](https://github.com/withastro/astro-discovery/issues) +- **ActivityPub/Mastodon**: Federated social networks +- **OpenID Connect**: Identity federation +- **Team discovery**: Find team members across services +- **Resource metadata**: Link identities to profiles + +## Step 1: Enable WebFinger + +Open your `astro.config.mjs` and enable WebFinger: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + webfinger: { + enabled: true, // WebFinger is opt-in + } + }) + ] +}); +``` + +Start dev server: + +```bash +npm run dev +``` + +Visit: `http://localhost:4321/.well-known/webfinger?resource=test` + +You'll see an empty response - let's add resources! + +## Step 2: Add Your First Resource + +Let's make yourself discoverable: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + webfinger: { + enabled: true, + resources: [ + { + resource: 'acct:you@your-site.com', + aliases: ['https://your-site.com/@you'], + links: [ + { + rel: 'http://webfinger.net/rel/profile-page', + type: 'text/html', + href: 'https://your-site.com/@you', + } + ], + } + ], + } + }) + ] +}); +``` + +Test it: + +```bash +npm run dev +``` + +Visit: `http://localhost:4321/.well-known/webfinger?resource=acct:you@your-site.com` + +You'll see: + +```json +{ + "subject": "acct:you@your-site.com", + "aliases": ["https://your-site.com/@you"], + "links": [ + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": "https://your-site.com/@you" + } + ] +} +``` + +Your first WebFinger resource! + +## Step 3: Add ActivityPub Support + +Make yourself discoverable on Mastodon: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + webfinger: { + enabled: true, + resources: [ + { + resource: 'acct:you@your-site.com', + aliases: [ + 'https://your-site.com/@you', + 'https://your-site.com/users/you', + ], + links: [ + { + rel: 'http://webfinger.net/rel/profile-page', + type: 'text/html', + href: 'https://your-site.com/@you', + }, + { + rel: 'self', + type: 'application/activity+json', // ActivityPub! + href: 'https://your-site.com/users/you', + } + ], + } + ], + } + }) + ] +}); +``` + +Build and test - Mastodon can now discover your profile! + +## Step 4: Add Multiple Team Members + +Let's add more people: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + webfinger: { + enabled: true, + resources: [ + { + resource: 'acct:alice@your-site.com', + properties: { + 'http://schema.org/name': 'Alice Developer', + 'http://schema.org/jobTitle': 'Lead Developer', + }, + links: [ + { + rel: 'http://webfinger.net/rel/profile-page', + href: 'https://your-site.com/team/alice', + }, + { + rel: 'self', + type: 'application/activity+json', + href: 'https://your-site.com/users/alice', + }, + ], + }, + { + resource: 'acct:bob@your-site.com', + properties: { + 'http://schema.org/name': 'Bob Designer', + 'http://schema.org/jobTitle': 'UX Designer', + }, + links: [ + { + rel: 'http://webfinger.net/rel/profile-page', + href: 'https://your-site.com/team/bob', + }, + ], + }, + ], + } + }) + ] +}); +``` + +Test both: +- `?resource=acct:alice@your-site.com` +- `?resource=acct:bob@your-site.com` + +## Step 5: Add Avatar Links + +Link to profile pictures: + +```typescript +{ + resource: 'acct:alice@your-site.com', + links: [ + { + rel: 'http://webfinger.net/rel/profile-page', + href: 'https://your-site.com/team/alice', + }, + { + rel: 'http://webfinger.net/rel/avatar', + type: 'image/jpeg', + href: 'https://your-site.com/avatars/alice.jpg', + }, + { + rel: 'self', + type: 'application/activity+json', + href: 'https://your-site.com/users/alice', + }, + ], +} +``` + +Now avatars are discoverable! + +## Step 6: Use Content Collections + +Automatically generate resources from Astro content collections: + +```typescript +export default defineConfig({ + site: 'https://your-site.com', + integrations: [ + discovery({ + webfinger: { + enabled: true, + collections: [ + { + name: 'team', // Your content collection + resourceTemplate: 'acct:{slug}@your-site.com', + linksBuilder: (member) => [ + { + rel: 'http://webfinger.net/rel/profile-page', + href: `https://your-site.com/team/${member.slug}`, + type: 'text/html', + }, + { + rel: 'http://webfinger.net/rel/avatar', + href: member.data.avatar, + type: 'image/jpeg', + }, + { + rel: 'self', + type: 'application/activity+json', + href: `https://your-site.com/users/${member.slug}`, + }, + ], + propertiesBuilder: (member) => ({ + 'http://schema.org/name': member.data.name, + 'http://schema.org/jobTitle': member.data.role, + }), + } + ], + } + }) + ] +}); +``` + +Now all team members from your content collection are automatically discoverable! + +## Step 7: Filter Links with Rel Parameter + +WebFinger supports filtering by link relation: + +Test: `?resource=acct:alice@your-site.com&rel=self` + +Only links with `rel="self"` will be returned! + +## What You've Learned + +You now know how to: +- Enable WebFinger on your site +- Create discoverable resources +- Add ActivityPub/Mastodon support +- Link profile pages and avatars +- Add semantic properties +- Use content collections for dynamic resources +- Filter results with rel parameter + +## Complete Example: ActivityPub Site + +Here's a full setup for a federated social site: + +```typescript +import { defineConfig } from 'astro'; +import discovery from '@astrojs/discovery'; + +export default defineConfig({ + site: 'https://social-site.com', + integrations: [ + discovery({ + webfinger: { + enabled: true, + + // Static resources + resources: [ + { + resource: 'acct:admin@social-site.com', + subject: 'acct:admin@social-site.com', + aliases: [ + 'https://social-site.com/@admin', + 'https://social-site.com/users/admin', + ], + properties: { + 'http://schema.org/name': 'Site Admin', + }, + links: [ + { + rel: 'http://webfinger.net/rel/profile-page', + type: 'text/html', + href: 'https://social-site.com/@admin', + }, + { + rel: 'self', + type: 'application/activity+json', + href: 'https://social-site.com/users/admin', + }, + { + rel: 'http://webfinger.net/rel/avatar', + type: 'image/png', + href: 'https://social-site.com/avatars/admin.png', + }, + ], + }, + ], + + // Dynamic from content collection + collections: [ + { + name: 'users', + resourceTemplate: 'acct:{slug}@social-site.com', + aliasesBuilder: (user) => [ + `https://social-site.com/@${user.slug}`, + `https://social-site.com/users/${user.slug}`, + ], + linksBuilder: (user) => [ + { + rel: 'http://webfinger.net/rel/profile-page', + type: 'text/html', + href: `https://social-site.com/@${user.slug}`, + }, + { + rel: 'self', + type: 'application/activity+json', + href: `https://social-site.com/users/${user.slug}`, + }, + { + rel: 'http://webfinger.net/rel/avatar', + type: user.data.avatarType || 'image/jpeg', + href: user.data.avatar, + }, + ], + propertiesBuilder: (user) => ({ + 'http://schema.org/name': user.data.displayName, + 'http://schema.org/description': user.data.bio, + }), + }, + ], + } + }) + ] +}); +``` + +## Use Case Examples + +### Mastodon/ActivityPub Integration + +```typescript +{ + resource: 'acct:user@example.com', + links: [ + { + rel: 'self', + type: 'application/activity+json', + href: 'https://example.com/users/user' + } + ] +} +``` + +Mastodon will query your WebFinger, then fetch the ActivityPub actor! + +### OpenID Connect + +```typescript +{ + resource: 'acct:user@example.com', + links: [ + { + rel: 'http://openid.net/specs/connect/1.0/issuer', + href: 'https://example.com' + } + ] +} +``` + +Enables identity federation! + +### Team Directory + +```typescript +{ + resource: 'acct:support@example.com', + properties: { + 'http://schema.org/name': 'Support Team', + 'http://schema.org/email': 'support@example.com' + }, + links: [ + { + rel: 'http://webfinger.net/rel/profile-page', + href: 'https://example.com/support' + } + ] +} +``` + +## Testing Your WebFinger + +### Manual Testing + +```bash +curl "https://your-site.com/.well-known/webfinger?resource=acct:you@your-site.com" +``` + +### With Mastodon + +1. Open Mastodon +2. Search for `you@your-site.com` +3. If WebFinger and ActivityPub are configured, you'll be discoverable! + +### With WebFinger Client + +Use a WebFinger client library to test programmatically. + +## Common Link Relations + +- `http://webfinger.net/rel/profile-page` - HTML profile page +- `self` - The resource itself (ActivityPub actor) +- `http://webfinger.net/rel/avatar` - Profile picture +- `http://openid.net/specs/connect/1.0/issuer` - OpenID issuer +- `http://ostatus.org/schema/1.0/subscribe` - Subscription endpoint + +## Troubleshooting + +### WebFinger Not Working? + +Make sure: + +1. **Enabled**: `enabled: true` in config +2. **Resources added**: At least one resource or collection +3. **Query parameter**: Include `?resource=acct:user@domain.com` +4. **CORS**: WebFinger includes CORS headers automatically + +### Resource Not Found? + +Check the exact resource URI: + +```bash +curl "http://localhost:4321/.well-known/webfinger?resource=acct:exact@match.com" +``` + +Resource must match exactly! + +### Mastodon Can't Find You? + +Mastodon needs: +1. WebFinger with correct resource format +2. ActivityPub actor at the linked URL +3. Proper CORS headers (automatic) +4. HTTPS in production + +### Content Collection Not Working? + +Make sure: +- Collection name matches exactly +- Template variables use correct field names +- linksBuilder and propertiesBuilder return correct types + +## Security Considerations + +### Privacy + +WebFinger makes information public. Only expose what you want discoverable: +- Don't include private email addresses +- Be careful with personal information +- Consider what's already public + +### Rate Limiting + +WebFinger endpoints can be queried frequently. Consider: +- Caching responses +- Rate limiting at server level +- CDN caching + +### Validation + +The integration validates resource URIs automatically, but be careful with: +- User-generated content in collections +- External URLs in links +- Property values + +## Next Steps + +- [ActivityPub How-To](/how-to/activitypub/) - Full ActivityPub integration +- [Content Collections How-To](/how-to/content-collections/) - Advanced collection patterns +- [WebFinger Explained](/explanation/webfinger-explained/) - Deep dive into RFC 7033 + +## Additional Resources + +- [RFC 7033 - WebFinger](https://www.rfc-editor.org/rfc/rfc7033.html) +- [WebFinger.net](https://webfinger.net/) +- [ActivityPub Spec](https://www.w3.org/TR/activitypub/) +- [Mastodon Documentation](https://docs.joinmastodon.org/) diff --git a/status.json b/status.json new file mode 100644 index 0000000..faf0864 --- /dev/null +++ b/status.json @@ -0,0 +1,101 @@ +{ + "task_masters": { + "tutorials": { + "status": "ready", + "branch": "docs/tutorials-content", + "worktree": "docs-tutorials", + "pages": [ + "getting-started/installation.md", + "getting-started/quick-start.md", + "getting-started/first-steps.md", + "tutorials/basic-setup.md", + "tutorials/configure-robots.md", + "tutorials/setup-llms.md", + "tutorials/create-humans.md", + "tutorials/security-canary.md", + "tutorials/webfinger.md" + ], + "dependencies": [], + "completed_pages": [ + "getting-started/installation.md", + "getting-started/quick-start.md", + "getting-started/first-steps.md", + "tutorials/basic-setup.md", + "tutorials/configure-robots.md", + "tutorials/setup-llms.md", + "tutorials/create-humans.md", + "tutorials/security-canary.md", + "tutorials/webfinger.md" + ] + }, + "howto": { + "status": "executing", + "branch": "docs/howto-content", + "worktree": "docs-howto", + "pages": [ + "how-to/block-bots.md", + "how-to/customize-llm-instructions.md", + "how-to/add-team-members.md", + "how-to/filter-sitemap.md", + "how-to/cache-headers.md", + "how-to/environment-config.md", + "how-to/content-collections.md", + "how-to/custom-templates.md", + "how-to/activitypub.md" + ], + "dependencies": ["tutorials"], + "completed_pages": [] + }, + "reference": { + "status": "executing", + "branch": "docs/reference-content", + "worktree": "docs-reference", + "pages": [ + "reference/configuration.md", + "reference/api.md", + "reference/robots.md", + "reference/llms.md", + "reference/humans.md", + "reference/security.md", + "reference/canary.md", + "reference/webfinger.md", + "reference/sitemap.md", + "reference/cache.md", + "reference/typescript.md" + ], + "dependencies": [], + "completed_pages": [] + }, + "explanation": { + "status": "executing", + "branch": "docs/explanation-content", + "worktree": "docs-explanation", + "pages": [ + "explanation/why-discovery.md", + "explanation/robots-explained.md", + "explanation/llms-explained.md", + "explanation/humans-explained.md", + "explanation/security-explained.md", + "explanation/canary-explained.md", + "explanation/webfinger-explained.md", + "explanation/seo.md", + "explanation/ai-integration.md", + "explanation/architecture.md", + "examples/ecommerce.md", + "examples/documentation.md", + "examples/blog.md", + "examples/api-platform.md", + "examples/multilanguage.md", + "examples/federated-social.md", + "community/contributing.md", + "community/changelog.md", + "community/troubleshooting.md", + "community/faq.md" + ], + "dependencies": [], + "completed_pages": [] + } + }, + "merge_order": ["reference", "tutorials", "howto", "explanation"], + "integration_status": "pending" +}