Do you really need a full blown (PHP) CMS for your personal Website? Here's a script that says no.
The recent PHP vulnerability hit my wife harder than me - I had "converted" to static HTML a while ago, while she was still running a dinosaur CMS for her sites. OK, I admit, my personal site is still on a CMS, but that
The recent PHP vulnerability hit my wife harder than me - I had "converted" to static HTML a while ago, while she was still running a dinosaur CMS for her sites.
OK, I admit, my personal site is still on a CMS, but that site runs content from 2008, so it might be up for a change anyway.
For those of you who, like me, really feel like not having the time to update PHP and then update it again, so that you can update it when the next update updates the updates ... and somehow nothings really works, why not pull the plug and go static html?
This script - yes, it's dirty, cheap and clobbered together - has converted a dozen sites for me in a few minutes. What it does is it calls the main entry point of BASE_URL and then "simply walks along", trying to get any local content copied to your local "static_site" directory. It updates, along the way, any auto-generated content (say, Drupal's "/node/x") to a local HTML ("/node/x.html") both on file and in file.
There most probably are many improvements possible, but it works fine here and has spared my at least two days of manual work.
import os
import re
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
import requests
# Configuration
# Point this to your local server (e.g., http://localhost/my_php_site/ or http://localhost/ if at root)
BASE_URL = "http://wherever.whatever.whoever/"
OUTPUT_DIR = "./static_site"
visited_urls = set()
def same_site(a, b):
"""Treat www.example.com and example.com as the same site."""
na = urlparse(a).netloc.lower().removeprefix("www.")
nb = urlparse(b).netloc.lower().removeprefix("www.")
return na == nb
def get_absolute_url(base_url, href):
"""Robustly resolves links, ensuring subfolder paths aren't lost on root-relative links."""
if not href:
return None
href = href.split("#")[0].strip()
if not href or href.startswith(("javascript:", "mailto:", "tel:")):
return None
parsed_base = urlparse(BASE_URL)
base_path = parsed_base.path.rstrip("/") # e.g., "/my_php_site"
# If it's already a full http/https URL
if href.startswith("http://") or href.startswith("https://"):
return href
# If it's root-relative (starts with /) and base_path exists, ensure subfolder is included for fetching
if href.startswith("/") and base_path and not href.startswith(base_path):
return f"{parsed_base.scheme}://{parsed_base.netloc}{base_path}{href}"
return urljoin(base_url, href)
def get_relative_subpath(url, base_url):
"""Extracts the clean relative route inside the site (e.g., node/3 -> node/3.html)."""
parsed_url = urlparse(url)
parsed_base = urlparse(base_url)
if parsed_url.netloc and not same_site(url, base_url):
return None # External link
path = parsed_url.path
base_path_prefix = parsed_base.path.rstrip("/")
# Strip base path prefix if present
if base_path_prefix and path.startswith(base_path_prefix):
path = path[len(base_path_prefix) :]
path = path.lstrip("/")
# Handle root, directories, .php files, and extensionless dynamic routes (like node/3)
if path == "" or path.endswith("/"):
path = path + "index.html"
elif path.endswith(".php"):
path = path[:-4] + ".html"
elif not os.path.splitext(path)[1] and not path.lower().endswith(
(
".css",
".js",
".png",
".jpg",
".jpeg",
".gif",
".pdf",
".svg",
".ico",
".json",
".xml",
)
):
path = path + ".html"
return path
def get_local_path(url, base_url, output_dir):
"""Determines the absolute local file path for saving."""
subpath = get_relative_subpath(url, base_url)
if not subpath:
return None
return os.path.join(output_dir, subpath)
def get_root_relative_url(url, base_url):
"""Converts any internal URL into a clean root-relative path (e.g., /node/3.html)."""
subpath = get_relative_subpath(url, base_url)
if not subpath:
return url # External link
return "/" + subpath
CSS_URL_RE = re.compile(r"""url\(\s*(["']?)([^)"']+)\1\s*\)""")
CSS_IMPORT_RE = re.compile(r"""@import\s+(["'])([^"']+)\1""")
def map_css_ref(raw, css_url, to_crawl):
"""Returns the rewritten root-relative URL for a CSS reference, or None to leave it untouched."""
raw = raw.strip()
if not raw or raw.startswith("data:"):
return None
absolute = get_absolute_url(css_url, raw)
if not absolute or not same_site(absolute, BASE_URL):
return None
to_crawl.append(absolute)
new_url = get_root_relative_url(absolute, BASE_URL)
query = urlparse(raw).query
if query:
new_url += f"?{query}"
return new_url
def rewrite_css(css_text, css_url, to_crawl):
"""Rewrites url(...) and @import references and collects the originals for crawling."""
def repl_url(m):
new = map_css_ref(m.group(2), css_url, to_crawl)
return m.group(0) if new is None else f"url({m.group(1)}{new}{m.group(1)})"
def repl_import(m):
new = map_css_ref(m.group(2), css_url, to_crawl)
return m.group(0) if new is None else f"@import {m.group(1)}{new}{m.group(1)}"
css_text = CSS_URL_RE.sub(repl_url, css_text)
return CSS_IMPORT_RE.sub(repl_import, css_text)
def process_and_save(url):
"""Crawls the site, saves dynamic routes into subdirectories, and rewrites links to root-relative."""
if url in visited_urls:
return
visited_urls.add(url)
print(f"Processing: {url}")
try:
response = requests.get(url, timeout=30)
if response.status_code != 200:
print(f"Skipping {url} (Status code: {response.status_code})")
return
content_type = response.headers.get("content-type", "").lower()
local_path = get_local_path(url, BASE_URL, OUTPUT_DIR)
if not local_path:
return
os.makedirs(os.path.dirname(local_path), exist_ok=True)
# Check if it's an HTML page / dynamic route or a static asset (CSS, JS, Image, PDF)
if "text/html" in content_type or not os.path.splitext(urlparse(url).path)[1]:
html_content = response.text
soup = BeautifulSoup(html_content, "html.parser")
link_mappings = [
("a", "href"),
("link", "href"),
("script", "src"),
("img", "src"),
("source", "src"),
("form", "action"),
]
to_crawl=[]
# First pass: Rewrite internal links to root-relative format (e.g., /node/3.html)
for tag_name, attr in link_mappings:
for tag in soup.find_all(tag_name, **{attr: True}):
attr_val = tag[attr]
absolute_url = get_absolute_url(url, attr_val)
if absolute_url and same_site(absolute_url, BASE_URL):
to_crawl.append(absolute_url)
new_url = get_root_relative_url(absolute_url, BASE_URL)
# Preserve query parameters if they existed
parsed_orig = urlparse(attr_val)
if parsed_orig.query:
new_url += f"?{parsed_orig.query}"
tag[attr] = new_url
# Handle <style> blocks (Drupal puts @import url(...) in here)
for style_tag in soup.find_all("style"):
if style_tag.string:
style_tag.string.replace_with(rewrite_css(style_tag.string, url, to_crawl))
# Save the modified HTML file
with open(local_path, "w", encoding="utf-8") as f:
f.write(str(soup))
print(f"Saved HTML -> {local_path}")
# Second pass: crawl the ORIGINAL remote URLs collected above
skip_extensions = (".mp4", ".webm", ".avi", ".mov", ".mp3", ".wav")
for target in to_crawl:
if target in visited_urls:
continue
if urlparse(target).path.lower().endswith(skip_extensions):
continue
process_and_save(target)
elif "text/css" in content_type or urlparse(url).path.lower().endswith(".css"):
# CSS: rewrite internal references, then crawl what they point to
css_targets = []
css_text = response.content.decode("utf-8", errors="replace")
css_text = rewrite_css(css_text, url, css_targets)
with open(local_path, "w", encoding="utf-8") as f:
f.write(css_text)
print(f"Saved CSS -> {local_path}")
for target in css_targets:
if target not in visited_urls:
process_and_save(target)
else:
# Save static binary assets (JS, images, PDFs, etc.)
with open(local_path, "wb") as f:
f.write(response.content)
print(f"Saved Asset -> {local_path}")
except Exception as e:
print(f"Error processing {url}: {e}")
if __name__ == "__main__":
os.makedirs(OUTPUT_DIR, exist_ok=True)
print(f"Starting static conversion for: {BASE_URL}")
process_and_save(BASE_URL)
print(
f"\nDone! Fully functional static site generated inside:"
f" {os.path.abspath(OUTPUT_DIR)}"
)
Originally published by Dev.to WebDev. Aggregated on AIWithGhost for educational purposes — full credit and traffic to the original publisher.