Dev.to WebDev 🛠 Dev 👁 0 📖 6 min read

Do you really need a full blown (PHP) CMS for your personal Website? Here's a script that says no.

The recent PHP vulnerability hit my wife harder than me - I had "converted" to static HTML a while ago, while she was still running a dinosaur CMS for her sites. OK, I admit, my personal site is still on a CMS, but that

The recent PHP vulnerability hit my wife harder than me - I had "converted" to static HTML a while ago, while she was still running a dinosaur CMS for her sites.

OK, I admit, my personal site is still on a CMS, but that site runs content from 2008, so it might be up for a change anyway.

For those of you who, like me, really feel like not having the time to update PHP and then update it again, so that you can update it when the next update updates the updates ... and somehow nothings really works, why not pull the plug and go static html?

This script - yes, it's dirty, cheap and clobbered together - has converted a dozen sites for me in a few minutes. What it does is it calls the main entry point of BASE_URL and then "simply walks along", trying to get any local content copied to your local "static_site" directory. It updates, along the way, any auto-generated content (say, Drupal's "/node/x") to a local HTML ("/node/x.html") both on file and in file.

There most probably are many improvements possible, but it works fine here and has spared my at least two days of manual work.

import os
import re
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
import requests

# Configuration
# Point this to your local server (e.g., http://localhost/my_php_site/ or http://localhost/ if at root)
BASE_URL = "http://wherever.whatever.whoever/"
OUTPUT_DIR = "./static_site"

visited_urls = set()


def same_site(a, b):
  """Treat www.example.com and example.com as the same site."""
  na = urlparse(a).netloc.lower().removeprefix("www.")
  nb = urlparse(b).netloc.lower().removeprefix("www.")
  return na == nb

def get_absolute_url(base_url, href):
  """Robustly resolves links, ensuring subfolder paths aren't lost on root-relative links."""
  if not href:
    return None

  href = href.split("#")[0].strip()
  if not href or href.startswith(("javascript:", "mailto:", "tel:")):
    return None

  parsed_base = urlparse(BASE_URL)
  base_path = parsed_base.path.rstrip("/")  # e.g., "/my_php_site"

  # If it's already a full http/https URL
  if href.startswith("http://") or href.startswith("https://"):
    return href

  # If it's root-relative (starts with /) and base_path exists, ensure subfolder is included for fetching
  if href.startswith("/") and base_path and not href.startswith(base_path):
    return f"{parsed_base.scheme}://{parsed_base.netloc}{base_path}{href}"

  return urljoin(base_url, href)


def get_relative_subpath(url, base_url):
  """Extracts the clean relative route inside the site (e.g., node/3 -> node/3.html)."""
  parsed_url = urlparse(url)
  parsed_base = urlparse(base_url)

  if parsed_url.netloc and not same_site(url, base_url):
    return None  # External link

  path = parsed_url.path
  base_path_prefix = parsed_base.path.rstrip("/")

  # Strip base path prefix if present
  if base_path_prefix and path.startswith(base_path_prefix):
    path = path[len(base_path_prefix) :]

  path = path.lstrip("/")

  # Handle root, directories, .php files, and extensionless dynamic routes (like node/3)
  if path == "" or path.endswith("/"):
    path = path + "index.html"
  elif path.endswith(".php"):
    path = path[:-4] + ".html"
  elif not os.path.splitext(path)[1] and not path.lower().endswith(
      (
          ".css",
          ".js",
          ".png",
          ".jpg",
          ".jpeg",
          ".gif",
          ".pdf",
          ".svg",
          ".ico",
          ".json",
          ".xml",
      )
  ):
    path = path + ".html"

  return path


def get_local_path(url, base_url, output_dir):
  """Determines the absolute local file path for saving."""
  subpath = get_relative_subpath(url, base_url)
  if not subpath:
    return None
  return os.path.join(output_dir, subpath)


def get_root_relative_url(url, base_url):
  """Converts any internal URL into a clean root-relative path (e.g., /node/3.html)."""
  subpath = get_relative_subpath(url, base_url)
  if not subpath:
    return url  # External link
  return "/" + subpath

CSS_URL_RE = re.compile(r"""url\(\s*(["']?)([^)"']+)\1\s*\)""")
CSS_IMPORT_RE = re.compile(r"""@import\s+(["'])([^"']+)\1""")


def map_css_ref(raw, css_url, to_crawl):
  """Returns the rewritten root-relative URL for a CSS reference, or None to leave it untouched."""
  raw = raw.strip()
  if not raw or raw.startswith("data:"):
    return None
  absolute = get_absolute_url(css_url, raw)
  if not absolute or not same_site(absolute, BASE_URL):
    return None
  to_crawl.append(absolute)
  new_url = get_root_relative_url(absolute, BASE_URL)
  query = urlparse(raw).query
  if query:
    new_url += f"?{query}"
  return new_url


def rewrite_css(css_text, css_url, to_crawl):
  """Rewrites url(...) and @import references and collects the originals for crawling."""

  def repl_url(m):
    new = map_css_ref(m.group(2), css_url, to_crawl)
    return m.group(0) if new is None else f"url({m.group(1)}{new}{m.group(1)})"

  def repl_import(m):
    new = map_css_ref(m.group(2), css_url, to_crawl)
    return m.group(0) if new is None else f"@import {m.group(1)}{new}{m.group(1)}"

  css_text = CSS_URL_RE.sub(repl_url, css_text)
  return CSS_IMPORT_RE.sub(repl_import, css_text)

def process_and_save(url):
  """Crawls the site, saves dynamic routes into subdirectories, and rewrites links to root-relative."""
  if url in visited_urls:
    return
  visited_urls.add(url)

  print(f"Processing: {url}")
  try:
    response = requests.get(url, timeout=30)
    if response.status_code != 200:
      print(f"Skipping {url} (Status code: {response.status_code})")
      return

    content_type = response.headers.get("content-type", "").lower()
    local_path = get_local_path(url, BASE_URL, OUTPUT_DIR)
    if not local_path:
      return

    os.makedirs(os.path.dirname(local_path), exist_ok=True)

    # Check if it's an HTML page / dynamic route or a static asset (CSS, JS, Image, PDF)
    if "text/html" in content_type or not os.path.splitext(urlparse(url).path)[1]:
      html_content = response.text
      soup = BeautifulSoup(html_content, "html.parser")

      link_mappings = [
          ("a", "href"),
          ("link", "href"),
          ("script", "src"),
          ("img", "src"),
          ("source", "src"),
          ("form", "action"),
      ]

      to_crawl=[]
      # First pass: Rewrite internal links to root-relative format (e.g., /node/3.html)
      for tag_name, attr in link_mappings:
        for tag in soup.find_all(tag_name, **{attr: True}):
          attr_val = tag[attr]
          absolute_url = get_absolute_url(url, attr_val)

          if absolute_url and same_site(absolute_url, BASE_URL):
            to_crawl.append(absolute_url)
            new_url = get_root_relative_url(absolute_url, BASE_URL)

            # Preserve query parameters if they existed
            parsed_orig = urlparse(attr_val)
            if parsed_orig.query:
              new_url += f"?{parsed_orig.query}"

            tag[attr] = new_url

            # Handle <style> blocks (Drupal puts @import url(...) in here)
      for style_tag in soup.find_all("style"):
        if style_tag.string:
          style_tag.string.replace_with(rewrite_css(style_tag.string, url, to_crawl))

      # Save the modified HTML file

      with open(local_path, "w", encoding="utf-8") as f:
        f.write(str(soup))
      print(f"Saved HTML -> {local_path}")

      # Second pass: crawl the ORIGINAL remote URLs collected above
      skip_extensions = (".mp4", ".webm", ".avi", ".mov", ".mp3", ".wav")
      for target in to_crawl:
        if target in visited_urls:
          continue
        if urlparse(target).path.lower().endswith(skip_extensions):
          continue
        process_and_save(target)

    elif "text/css" in content_type or urlparse(url).path.lower().endswith(".css"):
      # CSS: rewrite internal references, then crawl what they point to
      css_targets = []
      css_text = response.content.decode("utf-8", errors="replace")
      css_text = rewrite_css(css_text, url, css_targets)
      with open(local_path, "w", encoding="utf-8") as f:
        f.write(css_text)
      print(f"Saved CSS -> {local_path}")
      for target in css_targets:
        if target not in visited_urls:
          process_and_save(target)

    else:
      # Save static binary assets (JS, images, PDFs, etc.)
      with open(local_path, "wb") as f:
        f.write(response.content)
      print(f"Saved Asset -> {local_path}")

  except Exception as e:
    print(f"Error processing {url}: {e}")


if __name__ == "__main__":
  os.makedirs(OUTPUT_DIR, exist_ok=True)
  print(f"Starting static conversion for: {BASE_URL}")
  process_and_save(BASE_URL)
  print(
      f"\nDone! Fully functional static site generated inside:"
      f" {os.path.abspath(OUTPUT_DIR)}"
  )
📰 Read the original article on Dev.to WebDev

Originally published by Dev.to WebDev. Aggregated on AIWithGhost for educational purposes — full credit and traffic to the original publisher.