Replace YouTube ingestion with Medium extraction

README.md

@@ -14,11 +14,11 @@ short_description: AI knowledge hub for groups, powered by Nvidia
 
 BuildSmall KnowledgeHub is a modular Gradio app for loading knowledge from:
 
-- YouTube links with public transcripts/captions
+- Medium article links through Freedium
 - arXiv links or IDs
 - PDF documents
 
-It extracts text, chunks it, embeds chunks locally with the configured NVIDIA Nemotron embedding model, uploads vectors into Qdrant, and generates grounded answers with NVIDIA's OpenAI-compatible chat API.
+It extracts text, captures Medium image references/captions when available, chunks the content, embeds chunks locally with the configured NVIDIA Nemotron embedding model, uploads vectors into Qdrant, and generates grounded answers with NVIDIA's OpenAI-compatible chat API.
 
 ## NVIDIA Usage
 
@@ -92,18 +92,15 @@ HF_TOKEN=<token-if-needed-for-gated-model-downloads>
 
 Use a hosted Qdrant instance for Hugging Face Spaces. `localhost:6333` only works for local development.
 
-## Hosted YouTube Transcript Note
+## Medium Article Extraction
 
-YouTube transcript fetching can fail on hosted platforms such as Hugging Face Spaces because YouTube may block transcript requests from datacenter IP addresses.
+Medium articles are fetched through Freedium:
 
-If a YouTube URL fails on the hosted app:
-
-1. Open the video on YouTube.
-2. Copy the transcript manually.
-3. Paste it into the **YouTube transcript fallback** box.
-4. Keep the YouTube URL in the URL field and run ingestion again.
+```text
+https://freedium-mirror.cfd/
+```
 
-This bypasses hosted transcript fetching while still storing the content as a YouTube source in Qdrant.
+Pass a Medium article URL into the app. The extractor builds a Freedium mirror URL, extracts the readable article text, collects image URLs and alt/caption text when available, then sends that combined content through the same chunking, embedding, and Qdrant upload pipeline.
 
 ## Qdrant Collection Name
 
@@ -141,10 +138,10 @@ The app binds to `0.0.0.0:7860`, which is suitable for Hugging Face Spaces and c
 ```text
 app/
   core/        settings and shared models
-  extractors/  PDF, arXiv, and YouTube extraction
+  extractors/  PDF, arXiv, and Medium extraction
   services/    chunking, embeddings, Qdrant, retrieval, ingestion orchestration
   ui/          Gradio Blocks UI
   utils/       source detection helpers
 ```
 
-YouTube extraction requires captions/transcripts to be available for the video. arXiv ingestion downloads the paper PDF and parses it with `pypdf`.
+Medium extraction uses `freedium-mirror.cfd`. arXiv ingestion downloads the paper PDF and parses it with `pypdf`.

app/core/models.py

@@ -7,7 +7,7 @@ from typing import Any
 class SourceType(str, Enum):
     PDF = "pdf"
     ARXIV = "arxiv"
-    YOUTUBE = "youtube"
+    MEDIUM = "medium"
 
 
 @dataclass(frozen=True)

app/extractors/medium.py

@@ -0,0 +1,126 @@
+from urllib.parse import quote, urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+from app.core.models import Document, SourceType
+
+
+FREEDIUM_BASE = "https://freedium-mirror.cfd"
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36"
+)
+
+
+def extract_medium(url: str) -> Document:
+    source_url = url.strip()
+    html, mirror_url = _fetch_freedium_html(source_url)
+    soup = BeautifulSoup(html, "html.parser")
+
+    for tag in soup(["script", "style", "noscript", "svg", "form", "nav", "header", "footer"]):
+        tag.decompose()
+
+    title = _extract_title(soup) or "Medium Article"
+    body = soup.find("article") or soup.find("main") or soup.body
+    if body is None:
+        raise ValueError("Freedium returned a page without readable article content.")
+
+    text_parts = _extract_text_parts(body)
+    image_parts = _extract_images(body, mirror_url)
+    combined = "\n\n".join([*text_parts, *image_parts]).strip()
+
+    if len(combined) < 300:
+        raise ValueError(
+            "Could not extract enough readable content from the Medium article through Freedium. "
+            "Check that the article URL is public and try again."
+        )
+
+    return Document(
+        source_type=SourceType.MEDIUM,
+        title=title,
+        text=combined,
+        source=source_url,
+        metadata={
+            "mirror_url": mirror_url,
+            "images": len(image_parts),
+            "extractor": "freedium-mirror.cfd",
+        },
+    )
+
+
+def _fetch_freedium_html(source_url: str) -> tuple[str, str]:
+    candidates = _freedium_candidates(source_url)
+    errors: list[str] = []
+    for candidate in candidates:
+        try:
+            response = requests.get(
+                candidate,
+                headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml"},
+                timeout=45,
+            )
+            response.raise_for_status()
+            if response.text.strip():
+                return response.text, response.url
+        except requests.RequestException as exc:
+            errors.append(f"{candidate}: {exc}")
+    raise ValueError("Could not fetch the Medium article through Freedium. " + " | ".join(errors[-2:]))
+
+
+def _freedium_candidates(source_url: str) -> list[str]:
+    parsed = urlparse(source_url)
+    if "freedium" in parsed.netloc:
+        return [source_url]
+    return [
+        f"{FREEDIUM_BASE}/{source_url}",
+        f"{FREEDIUM_BASE}/{quote(source_url, safe='')}",
+    ]
+
+
+def _extract_title(soup: BeautifulSoup) -> str:
+    for selector in ['meta[property="og:title"]', 'meta[name="twitter:title"]']:
+        tag = soup.select_one(selector)
+        if tag and tag.get("content"):
+            return tag["content"].strip()
+    heading = soup.find("h1")
+    if heading:
+        return heading.get_text(" ", strip=True)
+    if soup.title:
+        return soup.title.get_text(" ", strip=True)
+    return ""
+
+
+def _extract_text_parts(body) -> list[str]:
+    parts: list[str] = []
+    seen: set[str] = set()
+    for tag in body.find_all(["h1", "h2", "h3", "p", "li", "blockquote", "pre", "figcaption"]):
+        text = tag.get_text(" ", strip=True)
+        if not text or text in seen:
+            continue
+        seen.add(text)
+        if tag.name in {"h1", "h2", "h3"}:
+            parts.append(f"## {text}")
+        elif tag.name == "blockquote":
+            parts.append(f"> {text}")
+        else:
+            parts.append(text)
+    return parts
+
+
+def _extract_images(body, base_url: str) -> list[str]:
+    images: list[str] = []
+    seen: set[str] = set()
+    for image in body.find_all("img"):
+        src = image.get("src") or image.get("data-src") or image.get("data-original")
+        if not src:
+            continue
+        absolute_src = urljoin(base_url, src)
+        if absolute_src in seen:
+            continue
+        seen.add(absolute_src)
+        alt = image.get("alt", "").strip()
+        if alt:
+            images.append(f"Image: {alt}\nURL: {absolute_src}")
+        else:
+            images.append(f"Image URL: {absolute_src}")
+    return images

app/extractors/youtube.py

@@ -1,58 +0,0 @@
-from urllib.parse import parse_qs, urlparse
-
-from youtube_transcript_api import YouTubeTranscriptApi
-from youtube_transcript_api._errors import YouTubeTranscriptApiException
-
-from app.core.models import Document, SourceType
-
-
-def _extract_video_id(url: str) -> str:
-    parsed = urlparse(url.strip())
-    if parsed.netloc.endswith("youtu.be"):
-        return parsed.path.strip("/")
-    if "youtube.com" in parsed.netloc:
-        query = parse_qs(parsed.query)
-        if "v" in query:
-            return query["v"][0]
-        if parsed.path.startswith("/shorts/"):
-            return parsed.path.split("/")[2]
-    raise ValueError("Could not find a YouTube video ID in the URL.")
-
-
-def extract_youtube(url: str) -> Document:
-    video_id = _extract_video_id(url)
-    api = YouTubeTranscriptApi()
-    try:
-        if hasattr(api, "fetch"):
-            transcript = api.fetch(video_id)
-            transcript_items = transcript.to_raw_data()
-        else:
-            transcript_items = YouTubeTranscriptApi.get_transcript(video_id)
-    except YouTubeTranscriptApiException as exc:
-        raise ValueError(
-            "Could not fetch the YouTube transcript. On hosted environments such as "
-            "Hugging Face Spaces, YouTube often blocks transcript requests from datacenter IPs. "
-            "Paste the transcript into the YouTube transcript fallback box and retry."
-        ) from exc
-
-    if not transcript_items:
-        raise ValueError(
-            "No transcript was available for this YouTube video. Paste the transcript into "
-            "the YouTube transcript fallback box and retry."
-        )
-
-    lines = []
-    for item in transcript_items:
-        timestamp = int(item.get("start", 0))
-        minutes, seconds = divmod(timestamp, 60)
-        text = item.get("text", "").strip()
-        if text:
-            lines.append(f"[{minutes:02d}:{seconds:02d}] {text}")
-
-    return Document(
-        source_type=SourceType.YOUTUBE,
-        title=f"YouTube Transcript {video_id}",
-        text="\n".join(lines).strip(),
-        source=url,
-        metadata={"video_id": video_id, "segments": len(transcript_items)},
-    )

app/services/ingestion.py

@@ -3,8 +3,8 @@ from pathlib import Path
 from app.core.config import settings
 from app.core.models import Document, IngestionResult, SourceType
 from app.extractors.arxiv import extract_arxiv
+from app.extractors.medium import extract_medium
 from app.extractors.pdf import extract_pdf
-from app.extractors.youtube import extract_youtube
 from app.services.chat import NvidiaChatClient
 from app.services.chunking import chunk_document
 from app.services.embeddings import get_embedding_client
@@ -18,23 +18,14 @@ EXPORT_DIR = Path("data/exports")
 def extract_document(
     url: str | None = None,
     pdf_path: str | None = None,
-    manual_transcript: str | None = None,
 ) -> Document:
     source_type = detect_source(url, pdf_path)
     if source_type == SourceType.PDF:
         return extract_pdf(str(pdf_path))
     if source_type == SourceType.ARXIV:
         return extract_arxiv(str(url))
-    if source_type == SourceType.YOUTUBE:
-        if manual_transcript and manual_transcript.strip():
-            return Document(
-                source_type=SourceType.YOUTUBE,
-                title="YouTube Transcript",
-                text=manual_transcript.strip(),
-                source=str(url),
-                metadata={"transcript_source": "manual"},
-            )
-        return extract_youtube(str(url))
+    if source_type == SourceType.MEDIUM:
+        return extract_medium(str(url))
     raise ValueError(f"Unsupported source type: {source_type}")
 
 
@@ -69,9 +60,8 @@ def ingest_source(
     chunk_size: int | None = None,
     chunk_overlap: int | None = None,
     collection_name: str | None = None,
-    manual_transcript: str | None = None,
 ) -> IngestionResult:
-    document = extract_document(url=url, pdf_path=pdf_path, manual_transcript=manual_transcript)
+    document = extract_document(url=url, pdf_path=pdf_path)
     chunks = chunk_document(
         document,
         chunk_size=chunk_size or settings.CHUNK_SIZE,

app/ui/gradio_app.py

@@ -37,7 +37,6 @@ def _format_metadata(metadata: dict) -> str:
 def _ingest(
     url: str,
     pdf_file: str | None,
-    manual_transcript: str,
     collection_name: str,
 ):
     logger.info(
@@ -57,7 +56,6 @@ def _ingest(
             chunk_size=CHUNK_SIZE,
             chunk_overlap=CHUNK_OVERLAP,
             collection_name=collection_name,
-            manual_transcript=manual_transcript,
         )
         document = result.document
         status = (
@@ -174,7 +172,7 @@ def build_app() -> gr.Blocks:
             gr.Markdown(
                 f"""
 # {settings.PROJECT_NAME}
-Turn papers, PDFs, and videos into a searchable vector memory.
+Turn papers, PDFs, and articles into a searchable vector memory.
 
 Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded answers.
 """,
@@ -187,7 +185,7 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
   <div class="kh-chip">Parser <code>{settings.NEMOTRON_PARSE_MODEL}</code></div>
   <div class="kh-chip">Chat <code>{settings.NVIDIA_CHAT_MODEL}</code></div>
   <div class="kh-chip">Collection <code>{settings.QDRANT_COLLECTION_NAME}</code></div>
-  <div class="kh-chip">Sources PDF · arXiv · YouTube</div>
+  <div class="kh-chip">Sources PDF · arXiv · Medium</div>
 </div>
 """,
             )
@@ -200,8 +198,8 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
                                 "### Source Intake\n<div class='kh-subhead'>Upload a PDF or paste one link. The pipeline handles extraction, chunking, local embeddings, and Qdrant upload.</div>"
                             )
                             source_url = gr.Textbox(
-                                label="YouTube or arXiv input",
-                                placeholder="Paste a YouTube URL, arXiv URL, or arXiv ID",
+                                label="Medium or arXiv input",
+                                placeholder="Paste a Medium article URL, arXiv URL, or arXiv ID",
                                 lines=2,
                             )
                             pdf_file = gr.File(
@@ -209,11 +207,6 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
                                 file_types=[".pdf"],
                                 type="filepath",
                             )
-                            manual_transcript = gr.Textbox(
-                                label="YouTube transcript fallback",
-                                placeholder="If hosted YouTube transcript extraction is blocked, paste the transcript here and keep the YouTube URL above.",
-                                lines=5,
-                            )
                             collection_name_ingest = gr.Textbox(
                                 label="Collection Name",
                                 value=settings.QDRANT_COLLECTION_NAME,
@@ -263,7 +256,6 @@ Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded
                         inputs=[
                             source_url,
                             pdf_file,
-                            manual_transcript,
                             collection_name_ingest,
                         ],
                         outputs=[

app/utils/source_detection.py

@@ -6,7 +6,7 @@ from app.core.models import SourceType
 
 
 ARXIV_RE = re.compile(r"(?:arxiv\.org/(?:abs|pdf)/)?(?P<id>\d{4}\.\d{4,5})(?:v\d+)?", re.I)
-YOUTUBE_HOSTS = {"youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be", "www.youtu.be"}
+MEDIUM_HOST_PARTS = ("medium.com", "freedium")
 
 
 def detect_source(url: str | None, pdf_path: str | None) -> SourceType:
@@ -17,17 +17,17 @@ def detect_source(url: str | None, pdf_path: str | None) -> SourceType:
         raise ValueError("Uploaded file must be a PDF.")
 
     if not url or not url.strip():
-        raise ValueError("Provide a YouTube link, arXiv link/ID, or upload a PDF.")
+        raise ValueError("Provide a Medium article link, arXiv link/ID, or upload a PDF.")
 
     clean_url = url.strip()
     parsed = urlparse(clean_url)
     host = parsed.netloc.lower()
 
-    if host in YOUTUBE_HOSTS:
-        return SourceType.YOUTUBE
     if "arxiv.org" in host or ARXIV_RE.search(clean_url):
         return SourceType.ARXIV
-    raise ValueError("Could not detect source type. Use a YouTube URL, arXiv URL/ID, or PDF.")
+    if parsed.scheme in {"http", "https"}:
+        return SourceType.MEDIUM
+    raise ValueError("Could not detect source type. Use a Medium URL, arXiv URL/ID, or PDF.")
 
 
 def extract_arxiv_id(value: str) -> str:

pyproject.toml

@@ -1,11 +1,12 @@
 [project]
 name = "knowledgehub-ingestor"
 version = "1.0.0"
-description = "A Gradio document ingestion UI for PDFs, arXiv papers, and YouTube transcripts."
+description = "A Gradio document ingestion UI for PDFs, arXiv papers, and Medium articles."
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
     "arxiv>=2.1.3",
+    "beautifulsoup4>=4.12.3",
     "datasets>=5.0.0",
     "gradio>=5.0.0",
     "openai>=1.99.0",
@@ -17,7 +18,6 @@ dependencies = [
     "sentence-transformers>=3.0.1",
     "spaces",
     "torchvision>=0.27.0",
-    "youtube-transcript-api>=0.6.2",
 ]
 
 [project.optional-dependencies]

requirements.txt

@@ -1,4 +1,5 @@
 arxiv>=2.1.3
+beautifulsoup4>=4.12.3
 gradio>=5.0.0
 openai>=1.99.0
 pydantic-settings>=2.4.0
@@ -8,4 +9,3 @@ qdrant-client>=1.12.1
 requests>=2.32.3
 sentence-transformers>=3.0.1
 spaces
-youtube-transcript-api>=0.6.2