# Ruff Robot Website - Robots.txt
# Description: Search engine crawler instructions
# Author: James (Ruff Robot)
# Created: 2025-07-03

User-agent: *
Allow: /

# Block common spam and admin paths
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /.well-known/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/

# Allow important assets
Allow: /fonts/
Allow: /images/
Allow: /*.css
Allow: /*.js
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp
Allow: /*.ico
Allow: /*.mp4
Allow: /*.webm

# Block common unwanted files
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /search

# Sitemap location
Sitemap: https://ruffrobot.com/sitemap.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1