Audit on-page SEO for a whole site or sitemap: titles, canonicals, noindex and schema (Python + Node.js)

import re
import sys
import urllib.request
from html.parser import HTMLParser

class SEO(HTMLParser):
    def __init__(self):
        super().__init__()
        self.title, self.meta, self.h1, self.canonical = "", {}, 0, None
        self._in_title = False
    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "title":
            self._in_title = True
        elif tag == "meta" and a.get("name"):
            self.meta[a["name"].lower()] = a.get("content") or ""
        elif tag == "link" and (a.get("rel") or "").lower() == "canonical":
            self.canonical = a.get("href")
        elif tag == "h1":
            self.h1 += 1
    def handle_endtag(self, tag):
        if tag == "title":
            self._in_title = False
    def handle_data(self, data):
        if self._in_title:
            self.title += data

url = sys.argv[1] if len(sys.argv) > 1 else "https://www.scrapethissite.com/"
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (seo-check script)"})
with urllib.request.urlopen(req, timeout=20) as r:
    html = r.read().decode("utf-8", "replace")
    x_robots = r.headers.get("X-Robots-Tag", "")

p = SEO()
p.feed(html)
title, desc = p.title.strip(), p.meta.get("description", "")
issues = []
if not title: issues.append("Missing title")
elif not 30 <= len(title) <= 60: issues.append(f"Title length {len(title)} (aim for 30-60)")
if not desc: issues.append("Missing meta description")
elif not 70 <= len(desc) <= 160: issues.append(f"Description length {len(desc)} (aim for 70-160)")
if p.h1 != 1: issues.append(f"{p.h1} H1 headings (expected 1)")
if not p.canonical: issues.append("No canonical")
if "noindex" in (p.meta.get("robots", "") + x_robots).lower(): issues.append("noindex!")
if 'application/ld+json' not in html: issues.append("No JSON-LD")
print(url)
print("\n".join(f"  - {i}" for i in issues) or "  no issues")

python seo_check_stdlib.py https://your-site.example/

{"startUrls": ["https://www.example.com/"], "crawlSite": true, "maxPages": 200}

{"startUrls": ["https://www.example.com/sitemap.xml"], "maxPages": 1000, "onlyPagesWithIssues": true}

"""Run an on-page SEO audit of a site and print the worst pages and top issues.

Actor: https://apify.com/paperlinenstudio/seo-onpage-audit
Usage: APIFY_TOKEN=... python seo_audit.py https://your-site.example
"""
import os
import sys
from collections import Counter

from apify_client import ApifyClient

site = sys.argv[1] if len(sys.argv) > 1 else "https://www.scrapethissite.com/"
client = ApifyClient(os.environ["APIFY_TOKEN"])

run = client.actor("paperlinenstudio/seo-onpage-audit").call(
    run_input={
        "startUrls": [site],   # pages, a homepage or a sitemap.xml
        "crawlSite": True,     # follow internal links from the start URL
        "maxPages": 15,        # small crawl for the demo
    }
)
pages = list(client.dataset(run.default_dataset_id).iterate_items())
summary = client.key_value_store(run.default_key_value_store_id).get_record("SUMMARY")

print(f"{len(pages)} pages audited, average score {summary['value']['averageScore']}/100")
print("Lowest-scoring pages:")
for p in sorted(pages, key=lambda p: p["score"])[:5]:
    print(f"  {p['score']:>3}  {p['url']}")
print("Most common issues:")
counts = Counter(i.split(" (")[0] for p in pages for i in p["issues"])
for issue, n in counts.most_common(5):
    print(f"  {n:>3} x {issue}")

// Run an on-page SEO audit of a site and print the worst pages and top issues.
// Usage: APIFY_TOKEN=... node seo_audit.mjs https://your-site.example
import { ApifyClient } from 'apify-client';

const site = process.argv[2] ?? 'https://www.scrapethissite.com/';
const client = new ApifyClient({ token: process.env.APIFY_TOKEN });

const run = await client.actor('paperlinenstudio/seo-onpage-audit').call({
    startUrls: [site], // pages, a homepage or a sitemap.xml
    crawlSite: true,
    maxPages: 15,
});

const { items: pages } = await client.dataset(run.defaultDatasetId).listItems();
const summary = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('SUMMARY');

console.log(`${pages.length} pages audited, average score ${summary.value.averageScore}/100`);
console.log('Lowest-scoring pages:');
for (const p of [...pages].sort((a, b) => a.score - b.score).slice(0, 5)) {
    console.log(`  ${String(p.score).padStart(3)}  ${p.url}`);
}
const counts = {};
for (const p of pages) for (const i of p.issues) { const k = i.split(' (')[0]; counts[k] = (counts[k] ?? 0) + 1; }
console.log('Most common issues:');
for (const [issue, n] of Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 5)) {
    console.log(`  ${String(n).padStart(3)} x ${issue}`);
}

{
  "url": "https://www.heise.de/",
  "statusCode": 200,
  "responseTimeMs": 498,
  "blockedByRobotsTxt": false,
  "title": "heise online - IT-News, Nachrichten und Hintergründe | heise online",
  "titleLength": 67,
  "metaDescriptionLength": 140,
  "noindex": false,
  "canonical": "https://www.heise.de/",
  "canonicalIsSelf": true,
  "h1Count": 0,
  "h2Count": 23,
  "lang": "de",
  "viewport": true,
  "hreflang": [
    { "lang": "en", "url": "https://www.heise.de/en" },
    { "lang": "x-default", "url": "https://www.heise.de" }
  ],
  "jsonLdTypes": ["ItemList", "ListItem", "Thing"],
  "jsonLdErrors": 0,
  "wordCount": 3930,
  "imagesCount": 509,
  "imagesWithoutAlt": 32,
  "internalLinks": 279,
  "externalLinks": 102,
  "issues": [
    "Title too long (67 characters, may be truncated in Google; aim for 30-60)",
    "Missing H1 heading",
    "Images without alt attribute (32)"
  ],
  "score": 85
}