Ingesting YouTube Transcripts, Reddit Discussions & Documentation into AI / LLM RAG Pipelines (Python & Node.js)

export function extractPlayerResponse(html) {
    const markers = ['var ytInitialPlayerResponse = ', 'ytInitialPlayerResponse = '];
    let startIdx = -1;

    for (const marker of markers) {
        const pos = html.indexOf(marker);
        if (pos !== -1) {
            startIdx = html.indexOf('{', pos + marker.length - 1);
            break;
        }
    }
    if (startIdx === -1) return null;

    let depth = 0;
    let inString = false;
    let escaped = false;

    for (let i = startIdx; i < html.length; i++) {
        const ch = html[i];
        if (inString) {
            if (escaped) escaped = false;
            else if (ch === '\\') escaped = true;
            else if (ch === '"') inString = false;
        } else {
            if (ch === '"') inString = true;
            else if (ch === '{') depth++;
            else if (ch === '}') {
                depth--;
                if (depth === 0) {
                    try {
                        return JSON.parse(html.slice(startIdx, i + 1));
                    } catch {
                        return null;
                    }
                }
            }
        }
    }
    return null;
}

function decodeHtmlEntities(str) {
    if (!str) return '';
    return str
        .replace(/&/g, '&')
        .replace(/&lt;/g, '<')
        .replace(/&gt;/g, '>')
        .replace(/&quot;/g, '"')
        .replace(/&#39;/g, "'")
        .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
        .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)));
}

export async function fetchTranscriptSegments(playerResp, preferredLang = 'en') {
    const tracks = playerResp?.captions?.playerCaptionsTracklistRenderer?.captionTracks || [];
    const selected =
        tracks.find((t) => t.languageCode === preferredLang) ||
        tracks.find((t) => (t.languageCode || '').startsWith(preferredLang)) ||
        tracks[0];

    if (!selected?.baseUrl) return { language: null, segments: [] };

    const resp = await fetch(selected.baseUrl, {
        headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' },
    });
    if (!resp.ok) return { language: selected.languageCode, segments: [] };

    const xml = await resp.text();
    const segments = [];
    const regex = /<text\s+start="([\d.]+)"(?:\s+dur="([\d.]+)")?[^>]*>([\s\S]*?)<\/text>/g;

    for (const match of xml.matchAll(regex)) {
        const start = parseFloat(match[1]);
        const duration = match[2] ? parseFloat(match[2]) : 0;
        const text = decodeHtmlEntities(match[3].replace(/<[^>]+>/g, ''))
            .replace(/\s+/g, ' ')
            .trim();
        if (text) segments.push({ start, duration, text });
    }

    return { language: selected.languageCode, segments };
}

const words = (title + ' ' + selftext).split(/\s+/).filter(Boolean).length;
const markdownForAi = [
    `# ${title}`,
    `- **Subreddit**: ${subreddit} | **Author**: u/${author} | **Score**: ${score} | **Comments**: ${numComments}`,
    `- **URL**: ${permalink}`,
    '',
    selftext || '_External Link Post_',
].join('\n');

const record = {
    id,
    subreddit,
    title,
    author,
    score,
    numComments,
    url,
    permalink,
    selftext,
    wordCount: words,
    estimatedTokens: Math.ceil(words * 1.35),
    markdownForAi,
    scrapedAt: new Date().toISOString(),
};

import os
import requests
from typing import List, Dict

APIFY_TOKEN = os.getenv("APIFY_TOKEN", "YOUR_APIFY_TOKEN")

def run_apify_actor(actor_slug: str, payload: dict) -> List[Dict]:
    url = (
        f"https://api.apify.com/v2/acts/{actor_slug}/"
        f"run-sync-get-dataset-items?token={APIFY_TOKEN}"
    )
    resp = requests.post(url, json=payload, timeout=180)
    resp.raise_for_status()
    return resp.json()

def build_multi_source_rag_corpus() -> List[Dict]:
    corpus = []

    # 1. Ingest YouTube Video Transcripts & Metadata
    yt_items = run_apify_actor(
        "tungnguyen84~youtube-video-transcript-metadata-scraper",
        {
            "videoUrls": [{"url": "https://www.youtube.com/watch?v=jNQXAC9IVRw"}],
            "preferredLanguage": "en",
            "includeTimestamps": True,
            "maxVideos": 5,
        },
    )
    for item in yt_items:
        corpus.append({
            "source_type": "youtube",
            "url": item.get("url"),
            "title": item.get("title"),
            "tokens": item.get("estimatedTokens", 0),
            "markdown": item.get("markdownForAi", ""),
        })

    # 2. Ingest Reddit Subreddit Discussions
    reddit_items = run_apify_actor(
        "tungnguyen84~reddit-subreddit-json-ai-scraper",
        {
            "startUrls": [{"url": "https://old.reddit.com/r/programming/"}],
            "maxPosts": 15,
        },
    )
    for item in reddit_items:
        corpus.append({
            "source_type": "reddit",
            "url": item.get("permalink"),
            "title": item.get("title"),
            "tokens": item.get("estimatedTokens", 0),
            "markdown": item.get("markdownForAi", ""),
        })

    # 3. Ingest Technical Documentation Pages
    doc_items = run_apify_actor(
        "tungnguyen84~website-to-markdown-ai-rag-llm-scraper",
        {
            "startUrls": [{"url": "https://crawlee.dev"}],
            "maxPagesPerCrawl": 5,
        },
    )
    for item in doc_items:
        corpus.append({
            "source_type": "documentation",
            "url": item.get("url"),
            "title": item.get("title"),
            "tokens": item.get("estimatedTokens", 0),
            "markdown": item.get("markdown", ""),
        })

    return corpus

if __name__ == "__main__":
    documents = build_multi_source_rag_corpus()
    total_tokens = sum(d["tokens"] for d in documents)
    print(f"Ingested {len(documents)} clean RAG documents ({total_tokens} total tokens)")
    for doc in documents[:5]:
        print(f"[{doc['source_type'].upper()}] {doc['title']} ({doc['tokens']} tokens)")