User-agent: *
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/
Disallow: /404/
Disallow: /cookies/
Disallow: /takk/
Disallow: /privacy/


# AI crawlers — allow site content and machine-readable files
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /knowledge.json
Allow: /ai/
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /knowledge.json
Allow: /ai/
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /knowledge.json
Allow: /ai/
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/

User-agent: Anthropic-AI
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /knowledge.json
Allow: /ai/
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /knowledge.json
Allow: /ai/
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/

User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /knowledge.json
Allow: /ai/
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/

User-agent: Algolia Crawler
Disallow: /admin/
Disallow: /docs/
Disallow: /varsler/
Disallow: /tags/
Disallow: /atags/
Disallow: /kontakt/
Disallow: /404/
Disallow: /cookies/
Disallow: /takk/
Disallow: /privacy/

Sitemap: https://www.ulveseth.no/sitemap.xml