# robots.txt for MGN Egypt
# This file tells search engines which pages to crawl and which to ignore

# Allow all crawlers
User-agent: *
Allow: /

# Disallow admin and private areas (if applicable)
# Disallow: /admin/
# Disallow: /private/

# Specify sitemap location
Sitemap: https://mgneg.com/sitemap.xml

# Google-specific directives
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing-specific directives
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yandex-specific directives
User-agent: Yandex
Allow: /
Crawl-delay: 1

# Block bad bots
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /