Find Orphan Pages with Python (Standard Library Only): Compare Sitemap URLs with Internal Links, CSV Report

#!/usr/bin/env python3
"""Find orphan pages: URLs in your sitemap that no crawled page links to,
plus pages you can only reach from other orphans and linked pages that are
missing from the sitemap. CSV report.

Standard library only. Usage:
    python3 orphan_pages.py https://example.com/sitemap.xml --out orphans.csv
"""
import argparse
import csv
import gzip
import sys
import time
import urllib.error
import urllib.request
import urllib.robotparser
import xml.etree.ElementTree as ET
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from html.parser import HTMLParser
from urllib.parse import urldefrag, urljoin, urlsplit, urlunsplit

UA = "orphan-pages-audit/1.0 (+stdlib python)"
SM_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
SKIP_EXT = (".pdf", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".zip",
            ".mp4", ".mp3", ".css", ".js", ".xml", ".txt", ".ico", ".woff2")

def fetch(url, timeout=20, max_bytes=3_000_000):
    """GET a URL. Returns (status, final_url, content_type, body_bytes)."""
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            return (resp.status, resp.geturl(),
                    resp.headers.get("Content-Type", ""), resp.read(max_bytes))
    except urllib.error.HTTPError as e:
        return e.code, url, e.headers.get("Content-Type", "") if e.headers else "", b""
    except Exception as e:  # DNS, timeout, TLS, connection reset
        return None, url, "", str(e).encode()

def normalize(url, keep_query=True):
    """Exact form used for fetching: no fragment, lowercase scheme and host,
    no default port, '/' for an empty path."""
    url, _ = urldefrag(url.strip())
    p = urlsplit(url)
    host = (p.hostname or "").lower()
    if p.port and not ((p.scheme == "http" and p.port == 80) or
                       (p.scheme == "https" and p.port == 443)):
        host = f"{host}:{p.port}"
    return urlunsplit((p.scheme.lower(), host, p.path or "/",
                       p.query if keep_query else "", ""))

def match_key(url, strict_slash=False):
    """Looser form used only for matching sitemap URLs with links:
    '/about' and '/about/' count as the same page unless --strict-slash."""
    if strict_slash:
        return url
    p = urlsplit(url)
    path = p.path.rstrip("/") or "/"
    return urlunsplit((p.scheme, p.netloc, path, p.query, ""))

def load_sitemap(url, seen=None):
    """Return page URLs from a urlset or sitemap index (recursive, .gz ok)."""
    seen = seen if seen is not None else set()
    if url in seen:
        return []
    seen.add(url)
    status, _, _, body = fetch(url)
    if status != 200:
        raise RuntimeError(f"{url} returned {status or body.decode(errors='replace')}")
    if body[:2] == b"\x1f\x8b":
        body = gzip.decompress(body)
    root = ET.fromstring(body)
    locs = [el.text.strip() for el in root.iter(f"{SM_NS}loc") if el.text]
    if root.tag == f"{SM_NS}sitemapindex":
        urls = []
        for child in locs:
            urls.extend(load_sitemap(child, seen))
        return urls
    return locs

class LinkParser(HTMLParser):
    """Collect <a href> links the way Google describes crawlable links,
    plus <base href>, meta robots and rel=canonical."""

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.links = []          # (href, rel_nofollow)
        self.base = None
        self.robots = ""
        self.canonical = None
        self.script_only = 0     # <a> without href but with onclick/routerlink

    def handle_starttag(self, tag, attrs):
        a = {k.lower(): (v or "") for k, v in attrs}
        if tag == "a":
            if "href" in a:
                rel = a.get("rel", "").lower().split()
                self.links.append((a["href"], "nofollow" in rel))
            elif "onclick" in a or "routerlink" in a:
                self.script_only += 1
        elif tag == "base" and a.get("href") and self.base is None:
            self.base = a["href"]
        elif tag == "meta" and a.get("name", "").lower() in ("robots", "googlebot"):
            self.robots += " " + a.get("content", "").lower()
        elif tag == "link" and "canonical" in a.get("rel", "").lower().split():
            self.canonical = a.get("href")

class Page:
    def __init__(self, url):
        self.url = url
        self.status = None
        self.final = url
        self.html = False
        self.noindex = False
        self.canonical = ""
        self.depth = None        # click depth from start; None = not reached
        self.script_links = 0
        self.outlinks = []       # (target_url, nofollow)

def same_site(url, hosts):
    p = urlsplit(url)
    return p.scheme in ("http", "https") and p.netloc in hosts

def get_page(url, keep_query, delay):
    if delay:
        time.sleep(delay)
    page = Page(url)
    status, final, ctype, body = fetch(url)
    page.status = status
    page.final = normalize(final, keep_query)
    if status != 200 or "html" not in ctype.lower():
        return page
    page.html = True
    parser = LinkParser()
    charset = "utf-8"
    if "charset=" in ctype:
        charset = ctype.split("charset=")[-1].split(";")[0].strip() or "utf-8"
    try:
        parser.feed(body.decode(charset, errors="replace"))
    except LookupError:
        parser.feed(body.decode("utf-8", errors="replace"))
    base = urljoin(final, parser.base) if parser.base else final
    page.noindex = "noindex" in parser.robots or "none" in parser.robots.split()
    page_nofollow = "nofollow" in parser.robots or "none" in parser.robots.split()
    if parser.canonical:
        page.canonical = normalize(urljoin(base, parser.canonical), keep_query)
    page.script_links = parser.script_only
    for href, nofollow in parser.links:
        href = href.strip()
        if not href or href.lower().startswith(("javascript:", "mailto:", "tel:", "data:", "#")):
            continue
        page.outlinks.append((normalize(urljoin(base, href), keep_query),
                              nofollow or page_nofollow))
    return page

def main():
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("sitemap", help="sitemap.xml or sitemap index URL")
    ap.add_argument("--start", action="append",
                    help="crawl start URL (repeatable); default: site root")
    ap.add_argument("--out", default="orphan_report.csv")
    ap.add_argument("--max-pages", type=int, default=500,
                    help="stop the link crawl after this many fetches")
    ap.add_argument("--workers", type=int, default=4)
    ap.add_argument("--delay", type=float, default=0.0,
                    help="seconds to wait before each request (per worker)")
    ap.add_argument("--host", action="append", default=[],
                    help="extra host to treat as internal, e.g. www.example.com")
    ap.add_argument("--drop-query", action="store_true",
                    help="ignore ?query strings when comparing URLs")
    ap.add_argument("--strict-slash", action="store_true",
                    help="treat /page and /page/ as different URLs")
    ap.add_argument("--ignore-robots", action="store_true",
                    help="crawl paths robots.txt disallows (your own site only)")
    args = ap.parse_args()
    keep_q = not args.drop_query
    key = lambda u: match_key(u, args.strict_slash)

    # 1. Sitemap URLs
    try:
        raw = load_sitemap(args.sitemap)
    except Exception as e:
        print(f"Could not read sitemap: {e}")
        return 2
    sitemap = {}
    for u in raw:
        sitemap.setdefault(key(normalize(u, keep_q)), normalize(u, keep_q))
    root = urlunsplit(urlsplit(args.sitemap)._replace(path="/", query="", fragment=""))
    hosts = {urlsplit(root).netloc} | {h.lower() for h in args.host}
    starts = [normalize(s, keep_q) for s in (args.start or [root])]
    print(f"Sitemap: {len(sitemap)} URLs from {args.sitemap}")

    robots = urllib.robotparser.RobotFileParser(urljoin(root, "/robots.txt"))
    try:
        robots.read()
    except Exception:
        robots = None
    allowed = lambda u: args.ignore_robots or robots is None or robots.can_fetch(UA, u)

    pages = {}           # requested url -> Page
    blocked = set()

    def run(urls):
        with ThreadPoolExecutor(max_workers=args.workers) as pool:
            return list(pool.map(lambda u: get_page(u, keep_q, args.delay), urls))

    # 2. Breadth-first crawl from the start URLs, following <a href> links
    frontier, depth, hit_limit = [u for u in starts], 0, False
    queued = set(frontier)
    while frontier:
        batch = []
        for u in frontier:
            if not allowed(u):
                blocked.add(u)
            elif len(pages) + len(batch) < args.max_pages:
                batch.append(u)
            else:
                hit_limit = True
        nxt = []
        for page in run(batch):
            page.depth = depth
            pages[page.url] = page
            if page.final != page.url and page.final not in queued:
                queued.add(page.final)          # follow the redirect target too
                nxt.append(page.final)
            for target, nofollow in page.outlinks:
                if same_site(target, hosts) and not nofollow and target not in queued \
                        and not target.lower().endswith(SKIP_EXT):
                    queued.add(target)
                    nxt.append(target)
        frontier, depth = nxt, depth + 1
    reached = len(pages)

    # 3. Fetch sitemap URLs the crawl never reached, to read their status and links
    crawled = {key(p.url) for p in pages.values()}
    missing = [u for k, u in sitemap.items() if k not in crawled and allowed(u)]
    for page in run(missing):
        pages[page.url] = page          # depth stays None: not reachable from start

    # 4. Build the inbound link graph (keyed loosely, redirects resolved)
    redirect = {key(p.url): key(p.final) for p in pages.values() if p.final != p.url}
    resolve = lambda k: redirect.get(k, k)
    inlinks = defaultdict(set)          # target key -> source urls (followed)
    nofollow_in = defaultdict(set)
    via_redirect = defaultdict(set)
    depth_of = {}
    for p in pages.values():
        if p.depth is not None:
            for k in (key(p.url), key(p.final)):
                depth_of[k] = min(depth_of.get(k, p.depth), p.depth)
    for p in pages.values():
        for target, nofollow in p.outlinks:
            if not same_site(target, hosts):
                continue
            t = resolve(key(target))
            if t in (key(p.url), key(p.final)):
                continue                # a page linking to itself does not count
            (nofollow_in if nofollow else inlinks)[t].add(p.final)
            if t != key(target):
                via_redirect[t].add(p.final)
            if p.depth is not None and not nofollow:
                # files we skipped (PDF, .txt) still count as reached via this link
                depth_of[t] = min(depth_of.get(t, p.depth + 1), p.depth + 1)

    # 5. Classify
    by_key = {}
    for p in pages.values():
        by_key.setdefault(key(p.url), p)
    rows, counts = [], defaultdict(int)
    for k, url in sitemap.items():
        p = by_key.get(k)
        issues = []
        if p is None:                   # only skipped URLs were never fetched
            issues.append("blocked_by_robots")
        elif p is not None and p.status != 200:
            issues.append("http_error" if p.status else "fetch_failed")
        if p is not None and p.final != p.url:
            issues.append("redirected")
        fk = resolve(k)
        followed = inlinks.get(fk, set())
        nf = nofollow_in.get(fk, set())
        if not followed and not nf:
            issues.append("orphan")
        elif not followed:
            issues.append("nofollow_links_only")
        elif fk not in depth_of:
            issues.append("not_reached_from_start")
        if followed and via_redirect.get(fk) and via_redirect[fk] >= followed:
            issues.append("linked_only_via_redirect")
        if p is not None and p.noindex:
            issues.append("noindex")
        for i in issues:
            counts[i] += 1
        rows.append({
            "url": url, "in_sitemap": "yes",
            "status": p.status if p else "",
            "depth": depth_of.get(fk, ""),
            "inlinks": len(followed), "nofollow_inlinks": len(nf),
            "example_source": sorted(followed or nf)[0] if (followed or nf) else "",
            "issues": ";".join(issues),
        })

    # 6. Linked, indexable HTML pages that the sitemap does not list
    finals = {}
    for p in pages.values():
        if p.depth is not None and p.html and p.status == 200:
            finals.setdefault(key(p.final), p)
    for k, p in sorted(finals.items()):
        if k in sitemap or p.noindex:
            continue
        if p.canonical and key(p.canonical) != k:
            continue                    # canonicalised elsewhere: fine to leave out
        counts["not_in_sitemap"] += 1
        src = inlinks.get(k, set())
        rows.append({"url": p.final, "in_sitemap": "no", "status": p.status,
                     "depth": depth_of.get(k, ""), "inlinks": len(src),
                     "nofollow_inlinks": len(nofollow_in.get(k, set())),
                     "example_source": sorted(src)[0] if src else "",
                     "issues": "not_in_sitemap"})

    order = {"orphan": 0, "not_reached_from_start": 1, "nofollow_links_only": 2,
             "linked_only_via_redirect": 3, "not_in_sitemap": 4}
    rows.sort(key=lambda r: (min([order.get(i, 9) for i in r["issues"].split(";")] or [9]),
                             r["url"]))
    with open(args.out, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=list(rows[0].keys()) if rows else ["url"])
        w.writeheader()
        w.writerows(rows)

    script_pages = [p for p in pages.values() if p.script_links]
    print(f"Crawled {reached} pages from {', '.join(starts)}; "
          f"fetched {len(pages) - reached} sitemap URLs the crawl had not reached")
    if hit_limit:
        print(f"WARNING: stopped at --max-pages {args.max_pages}; "
              "orphan results are incomplete")
    if blocked:
        print(f"Skipped {len(blocked)} URLs disallowed by robots.txt")
    for name in ("orphan", "not_reached_from_start", "nofollow_links_only",
                 "linked_only_via_redirect", "not_in_sitemap", "redirected",
                 "http_error", "fetch_failed", "blocked_by_robots", "noindex"):
        if counts[name]:
            print(f"  {name:<26} {counts[name]}")
    if script_pages:
        print(f"  {len(script_pages)} pages have <a> tags with onclick/routerLink "
              "and no href (not counted as links)")
    for r in rows:
        if set(r["issues"].split(";")) & set(order):
            print(f"  {r['issues']:<40} {r['url']}")
    print(f"Report: {args.out}")
    problems = {"orphan", "not_reached_from_start", "nofollow_links_only", "http_error",
                "fetch_failed"}
    return 1 if any(counts[p] for p in problems) else 0

if __name__ == "__main__":
    sys.exit(main())

"""Serve ./site with clean URLs (/pricing -> pricing.html) and one redirect."""
import functools
import http.server
import os

REDIRECTS = {"/case-studies": "/customers"}

class Handler(http.server.SimpleHTTPRequestHandler):
    def do_GET(self):
        path = self.path.split("?")[0]
        if path in REDIRECTS:
            self.send_response(301)
            self.send_header("Location", REDIRECTS[path])
            self.end_headers()
            return
        if path.endswith("/"):           # /blog/ -> blog/index.html, /about/ -> about.html
            path = path + "index" if os.path.isdir("site" + path) else path.rstrip("/")
        if not os.path.splitext(path)[1]:
            path += ".html"
        self.path = path
        super().do_GET()

    def log_message(self, *args):
        pass

http.server.ThreadingHTTPServer(
    ("127.0.0.1", 8771),
    functools.partial(Handler, directory="site")).serve_forever()

python3 orphan_pages.py http://127.0.0.1:8771/sitemap.xml --out orphan_report.csv
echo "exit code: $?"

Sitemap: 14 URLs from http://127.0.0.1:8771/sitemap.xml
Crawled 13 pages from http://127.0.0.1:8771/; fetched 5 sitemap URLs the crawl had not reached
Skipped 1 URLs disallowed by robots.txt
  orphan                     3
  not_reached_from_start     1
  nofollow_links_only        1
  linked_only_via_redirect   1
  not_in_sitemap             1
  http_error                 1
  1 pages have <a> tags with onclick/routerLink and no href (not counted as links)
  orphan                                   http://127.0.0.1:8771/integrations/slack
  orphan                                   http://127.0.0.1:8771/landing/spring-promo
  http_error;orphan                        http://127.0.0.1:8771/webinar-2025
  not_reached_from_start                   http://127.0.0.1:8771/docs/setup
  nofollow_links_only                      http://127.0.0.1:8771/careers
  linked_only_via_redirect                 http://127.0.0.1:8771/customers
  not_in_sitemap                           http://127.0.0.1:8771/blog/post-3
Report: orphan_report.csv
exit code: 1

url,in_sitemap,status,depth,inlinks,nofollow_inlinks,example_source,issues
http://127.0.0.1:8771/integrations/slack,yes,200,,0,0,,orphan
http://127.0.0.1:8771/landing/spring-promo,yes,200,,0,0,,orphan
http://127.0.0.1:8771/webinar-2025,yes,404,,0,0,,http_error;orphan
http://127.0.0.1:8771/docs/setup,yes,200,,1,0,http://127.0.0.1:8771/landing/spring-promo,not_reached_from_start
http://127.0.0.1:8771/careers,yes,200,,0,15,http://127.0.0.1:8771/,nofollow_links_only
http://127.0.0.1:8771/customers,yes,200,1,1,0,http://127.0.0.1:8771/,linked_only_via_redirect
http://127.0.0.1:8771/blog/post-3,no,200,1,1,0,http://127.0.0.1:8771/,not_in_sitemap
http://127.0.0.1:8771/,yes,200,0,15,0,http://127.0.0.1:8771/about/,
http://127.0.0.1:8771/about,yes,200,1,15,0,http://127.0.0.1:8771/,
http://127.0.0.1:8771/blog/,yes,200,1,15,0,http://127.0.0.1:8771/,
http://127.0.0.1:8771/blog/post-1,yes,200,1,2,0,http://127.0.0.1:8771/,
http://127.0.0.1:8771/blog/post-2,yes,200,2,1,0,http://127.0.0.1:8771/blog/,
http://127.0.0.1:8771/features,yes,200,1,15,0,http://127.0.0.1:8771/,
http://127.0.0.1:8771/pricing,yes,200,1,15,0,http://127.0.0.1:8771/,
http://127.0.0.1:8771/privacy,yes,200,1,15,0,http://127.0.0.1:8771/,

Could not read sitemap: http://127.0.0.1:8771/missing.xml returned 404
exit code: 2

python3 orphan_pages.py https://example.com/sitemap.xml --out orphan_report.csv --workers 2 --limit 50