Find broken links on your website for free (and fail CI when new ones appear)

import sys
import urllib.request
from collections import defaultdict, deque
from concurrent.futures import ThreadPoolExecutor
from html.parser import HTMLParser
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urldefrag, urlparse

UA = {"User-Agent": "Mozilla/5.0 (link-check script)"}

class Links(HTMLParser):
    def __init__(self):
        super().__init__()
        self.links = []
    def handle_starttag(self, tag, attrs):
        if tag == "a":
            href = dict(attrs).get("href")
            if href:
                self.links.append(href)

def get(url, method="GET"):
    req = urllib.request.Request(url, headers=UA, method=method)
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            ctype = r.headers.get("Content-Type", "")
            body = r.read() if method == "GET" and "html" in ctype else b""
            return r.status, body
    except HTTPError as e:
        return e.code, b""
    except (URLError, TimeoutError) as e:
        return f"error: {getattr(e, 'reason', e)}", b""

def crawl(start, max_pages=20):
    host = urlparse(start).netloc
    found_on = defaultdict(set)          # link -> pages that contain it
    queue, seen, pages = deque([start]), {start}, 0
    while queue and pages < max_pages:
        page = queue.popleft()
        code, body = get(page)
        pages += 1
        p = Links()
        p.feed(body.decode("utf-8", "replace"))
        for href in p.links:
            link = urldefrag(urljoin(page, href))[0]
            if not link.startswith(("http://", "https://")):
                continue                 # mailto:, tel:, javascript:
            found_on[link].add(page)
            if urlparse(link).netloc == host and link not in seen:
                seen.add(link)
                queue.append(link)
    return found_on

def check(link):
    code, _ = get(link, "HEAD")
    if isinstance(code, str) or code >= 400:   # many servers mishandle HEAD: confirm with GET
        code, _ = get(link)
    return link, code

if __name__ == "__main__":
    site = sys.argv[1] if len(sys.argv) > 1 else "https://www.scrapethissite.com"
    found_on = crawl(site)
    with ThreadPoolExecutor(max_workers=8) as pool:
        results = list(pool.map(check, found_on))
    broken = [(l, c) for l, c in results if isinstance(c, str) or c >= 400]
    print(f"checked {len(results)} links, {len(broken)} broken")
    for link, code in broken:
        print(code, link, "<- found on", sorted(found_on[link])[0])
    sys.exit(1 if broken else 0)

python linkcheck_stdlib.py https://your-site.example

"""Crawl a website and list broken links, with the pages they were found on.

Actor: https://apify.com/paperlinenstudio/broken-link-checker
Usage: APIFY_TOKEN=... python broken_links.py https://example.com
"""
import os
import sys

from apify_client import ApifyClient

site = sys.argv[1] if len(sys.argv) > 1 else "https://www.scrapethissite.com"
client = ApifyClient(os.environ["APIFY_TOKEN"])

run = client.actor("paperlinenstudio/broken-link-checker").call(
    run_input={
        "startUrls": [{"url": site}],
        "maxPagesPerSite": 20,     # small crawl for the demo
        "outputMode": "issues",    # broken + redirects + blocked, skip OK links
    }
)
rows = list(client.dataset(run.default_dataset_id).iterate_items())

broken = [r for r in rows if r["result"] == "broken"]
redirects = [r for r in rows if r["result"] == "redirect"]
print(f"{len(broken)} broken link(s), {len(redirects)} redirect(s)")
for r in broken:
    page = r["foundOn"][0]["pageUrl"] if r["foundOn"] else "?"
    print(f"  {r['statusCode'] or r['errorType']}  {r['url']}  (found on {page})")
for r in redirects[:5]:
    hops = " -> ".join(str(h["statusCode"]) for h in r["redirectChain"])
    print(f"  redirect {hops}: {r['url']} => {r['finalUrl']}")

sys.exit(1 if broken else 0)  # handy in CI

2 broken link(s), 1 redirect(s)
  400  https://www.scrapethissite.com/pages/advanced/?gotcha=csrf  (found on https://www.scrapethissite.com/pages/advanced/)
  400  https://www.scrapethissite.com/pages/advanced/?gotcha=login  (found on https://www.scrapethissite.com/pages/advanced/)
  redirect 301: http://peric.github.io/GetCountries/ => https://peric.github.io/GetCountries/

// Crawl a site and report broken links plus the page each one was found on.
// Exits with code 1 when something is broken, so it works as a CI check.
// Usage: APIFY_TOKEN=... node broken_links.mjs https://your-site.example
import { ApifyClient } from 'apify-client';

const site = process.argv[2] ?? 'https://www.scrapethissite.com';
const client = new ApifyClient({ token: process.env.APIFY_TOKEN });

const run = await client.actor('paperlinenstudio/broken-link-checker').call({
    startUrls: [{ url: site }],
    maxPagesPerSite: 20,
    outputMode: 'issues', // broken + redirects
});

const { items } = await client.dataset(run.defaultDatasetId).listItems();
const broken = items.filter((i) => ['broken', 'timeout'].includes(i.result));
const redirects = items.filter((i) => i.result === 'redirect');

console.log(`${broken.length} broken link(s), ${redirects.length} redirect(s)`);
for (const b of broken) {
    console.log(`  ${b.statusCode ?? b.errorType}  ${b.url}  (found on ${b.foundOn?.[0]?.pageUrl})`);
}
for (const r of redirects.slice(0, 5)) {
    const hops = (r.redirectChain ?? []).map((h) => h.statusCode).join(' -> ');
    console.log(`  redirect ${hops}: ${r.url} => ${r.finalUrl}`);
}
process.exitCode = broken.length ? 1 : 0;

{
  "url": "https://www.scrapethissite.com/pages/advanced/?gotcha=csrf",
  "result": "broken",
  "statusCode": 400,
  "statusText": "Bad Request",
  "errorType": "http_400",
  "redirectChain": [],
  "isInternal": true,
  "linkType": "page",
  "responseTimeMs": 151,
  "foundOnCount": 1,
  "foundOn": [
    {
      "pageUrl": "https://www.scrapethissite.com/pages/advanced/",
      "anchorText": "CSRF & Hidden Values",
      "tag": "a"
    }
  ]
}

# Weekly broken-link check for your own website.
# Add a repository secret APIFY_TOKEN and set SITE_URL below.
name: Weekly link check
on:
  schedule:
    - cron: "0 6 * * 1"   # Mondays 06:00 UTC
  workflow_dispatch:
env:
  SITE_URL: https://example.com
jobs:
  links:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install -r python/requirements.txt
      - run: python python/broken_links.py "$SITE_URL"
        env:
          APIFY_TOKEN: {% raw %}${{ secrets.APIFY_TOKEN }}{% endraw %}

{
  "startUrls": [{ "url": "https://your-site.example" }],
  "maxPagesPerSite": 500,
  "checkExternalLinks": true,
  "checkResources": true,
  "excludePatterns": ["/wp-admin/", "?replytocom="],
  "outputMode": "broken"
}