#
# robots.txt for thinkdmg.com
# Managed by Digital Marketing Group — https://thinkdmg.com
# Purpose: Guide search engine bots and support AI-first SEO visibility
#

# ALLOW: All search engines can access public content
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /client-login/
Disallow: /private/
Disallow: /category/uncategorized/
Disallow: /tags/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/

# ALLOW: Access to uploads and optimized categories
Allow: /wp-content/uploads/

# STRATEGIC CONTENT CATEGORIES (LLM-friendly)
Allow: /category/brand-building/
Allow: /category/content-marketing/
Allow: /category/conversion-optimization/
Allow: /category/creativity/
Allow: /category/digital-marketing-faqs/
Allow: /category/digital-marketing-for-small-business/
Allow: /category/digital-marketing-trends/
Allow: /category/e-commerce-marketing/
Allow: /category/email-marketing/
Allow: /category/generative-engine-optimization/
Allow: /category/influencer-marketing/
Allow: /category/marketing/
Allow: /category/mobile/
Allow: /category/online-advertising/
Allow: /category/pay-per-click-ppc/
Allow: /category/seo/
Allow: /category/seo-strategies/
Allow: /category/social-media/
Allow: /category/video-marketing/
Allow: /category/web-analytics-and-data/

# BLOCKED BOTS (Known scrapers and non-beneficial crawlers)
User-agent: Nuclei
Disallow: /
User-agent: WikiDo
Disallow: /
User-agent: Riddler
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Zoominfobot
Disallow: /
User-agent: Go-http-client
Disallow: /
User-agent: Node/simplecrawler
Disallow: /
User-agent: CazoodleBot
Disallow: /
User-agent: dotbot/1.0
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: magpie-crawler
Disallow: /

# SITEMAP & AI POLICY
Sitemap: https://thinkdmg.com/sitemap_index.xml

# AI crawler policy reference (complements llms.txt)
Comment: See AI access rules → https://thinkdmg.com/llms.txt
Comment: AI crawler guide → https://thinkdmg.com/ai-crawler-guide/

# End of robots.txt