# robots.txt for DailyFileTools.com
# https://dailyfiletools.com
# Last updated: 2026-08-31

User-agent: *
Allow: /

# Disallow temporary and system directories
Disallow: /storage/
Disallow: /vendor/
Disallow: /bootstrap/
Disallow: /app/
Disallow: /config/
Disallow: /database/
Disallow: /routes/
Disallow: /resources/
Disallow: /tests/
Disallow: /.env
Disallow: /.git
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /package.json
Disallow: /package-lock.json
Disallow: /phpunit.xml
Disallow: /README.md

# Disallow file download routes (temporary files)
Disallow: /download/

# Disallow admin and system routes
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /password/
Disallow: /email/
Disallow: /api/
Disallow: /temp/
Disallow: /horizon/
Disallow: /telescope/
Disallow: /ignition/
Disallow: /filament/

# Disallow search and filter parameters to prevent duplicate content
Disallow: /*?*
Disallow: /*&*
Disallow: /search?

# Crawl delay for aggressive crawlers
Crawl-delay: 1

# Sitemap locations
Sitemap: https://dailyfiletools.com/sitemap.xml
Sitemap: https://dailyfiletools.com/llms.txt

# Allow specific search engines with custom rules
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Block aggressive crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /