# Rules for Twitter's crawler (Twitterbot)
User-agent: Twitterbot
Allow: /
Allow: /btcthumb4.png
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg

# Rules for all other crawlers
User-agent: *
Allow: /

# Explicitly disallow all API routes
Disallow: /api/
Disallow: /api/*

# Explicitly allow access to important resource files
Allow: /favicon.ico
Allow: /favicon-16x16.png
Allow: /favicon-32x32.png
Allow: /apple-touch-icon.png
Allow: /safari-pinned-tab.svg
Allow: /site.webmanifest
Allow: /btcthumb4.png
Allow: /social/*.png
Allow: /social/*.jpg

# Allow webmanifest files - previously blocked
Allow: /*.webmanifest

# Disallow crawling of generic resource files (except those explicitly allowed)
# Disallow: /*.ico$
# Disallow: /*.svg$
# Disallow: /*.png$
# Disallow: /*.jpg$
# Disallow: /*.jpeg$

# Disallow admin-related pages
Disallow: /admin/
Disallow: /auth/forgot-password

# Primary sitemap
Sitemap: https://bitcoinlaws.io/sitemap.xml

# Encourage frequent recrawling
Crawl-delay: 1