# robots.txt for educationincorporated.co.za (Education Incorporated / Edu Inc)
# Platform: WordPress
# Strategy: Allow full indexing of all public content and AI crawlers.
#            Block WordPress admin, security-sensitive, and duplicate-content paths.
# Deploy to: https://educationincorporated.co.za/robots.txt
# Last updated: 2026-09-11

# ===========================================================
# GENERAL RULES - all crawlers
# ===========================================================

User-agent: *
Allow: /

# -------------------------------------------------------
# WORDPRESS ADMIN & SECURITY PATHS - BLOCK ALL CRAWLERS
# These must never be indexed or exposed to the public web.
# wp-login.php and wp-admin are common brute-force targets.
# xmlrpc.php is a known attack vector - block unless you need it.
# -------------------------------------------------------
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-cron.php
Disallow: /wp-config.php

# Allow admin-ajax.php - required for WordPress AJAX features
# that power frontend functionality (contact forms, search, etc.)
Allow: /wp-admin/admin-ajax.php

# -------------------------------------------------------
# WORDPRESS CORE FILES - no SEO value, expose server info
# -------------------------------------------------------
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/upgrade/

# Allow uploads (images, PDFs) and themes (CSS, JS)
Allow: /wp-content/uploads/
Allow: /wp-content/themes/

# -------------------------------------------------------
# DUPLICATE & THIN CONTENT - avoid index pollution
# -------------------------------------------------------

# WordPress search results - thin, generated, not canonical
Disallow: /?s=
Disallow: /search/

# Author archive pages can expose WordPress usernames (security risk)
# and create near-duplicate content if posts are primarily from one author
Disallow: /?author=
Disallow: /author/

# Trackback endpoints - spam target, no SEO value
Disallow: /trackback/
Disallow: /*/trackback/

# Redundant index.php root (homepage is served at /)
Disallow: /index.php$

# Comment feeds - thin content, no SEO value
Disallow: /comments/feed/
Disallow: /*/comments/feed/

# Redundant tag and date archive pages (creates index bloat)
# Remove these Allow rules if you actively use tag/date archives
Disallow: /tag/
Disallow: /?p=

# WooCommerce personal/functional pages - not real search-result content
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# -------------------------------------------------------
# WORDPRESS REST API - exposes post IDs, user info, structure
# Block public REST API discovery to reduce information leakage
# -------------------------------------------------------
Disallow: /wp-json/wp/v2/users/
Disallow: /?rest_route=/wp/v2/users/

# Allow rest API for legitimate features if needed
Allow: /wp-json/

# -------------------------------------------------------
# GENERAL PRIVATE / BACKEND PATHS
# -------------------------------------------------------
Disallow: /private/
Disallow: /tmp/
Disallow: /logs/
Disallow: /cgi-bin/

# ===========================================================
# MAJOR SEARCH ENGINES - Full access to public content
# ===========================================================

User-agent: Googlebot
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /?s=
Disallow: /?author=
Disallow: /author/
Disallow: /tag/
Disallow: /?p=
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Allow: /

User-agent: Bingbot
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /?s=
Disallow: /?author=
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

User-agent: Slurp
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# ===========================================================
# AI & LLM CRAWLERS - Explicitly and fully allowed for all public content
#
# Education Incorporated's admissions funnel increasingly begins
# with parents asking AI assistants: "best small private school
# in Fourways", "IEB school with small classes Johannesburg",
# "boutique school Sandton for gifted child", etc.
# AI visibility for this school is a direct enrolment driver.
# ===========================================================

User-agent: GPTBot
Allow: /
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-includes/

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: Google-Extended
Allow: /
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /xmlrpc.php

# Meta AI - WhatsApp is widely used in SA school parent communities
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: Brave
Allow: /

# ===========================================================
# DISALLOW: SEO scrapers - bandwidth cost, no SEO value
# ===========================================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ===========================================================
# SITEMAP
# ===========================================================

Sitemap: https://educationincorporated.co.za/sitemap_index.xml
Sitemap: https://context.educationincorporated.co.za/sitemap.xml