import gradio as gr import pandas as pd import io import uuid import random import hashlib from datetime import datetime, timezone from huggingface_hub import hf_hub_download, HfApi import os DATASET_REPO = "meganariley/open-shelf" WRITE_TOKEN = os.environ.get("HF_TOKEN", os.environ.get("HF_WRITE_TOKEN", "")) GENRE_TAGS = ["Fiction","Non-Fiction","Mystery","Sci-Fi","Fantasy","Biography","History", "Romance","Thriller","Literary","Essays","Poetry","Graphic Novel","Self-Help","Travel"] MOOD_TAGS = ["Page-turner","Slow burn","Dense","Funny","Devastating","Uplifting", "Unsettling","Cozy","Challenging","Breezy","Cerebral","Emotional"] AVATAR_COLORS = ["#C1440E","#5C7A5F","#B8860B","#78716C","#44403C","#3182C8","#7B6EDE"] # ── Data layer ──────────────────────────────────────────────────────────────── _cache: dict = {} def load_books() -> pd.DataFrame: try: path = hf_hub_download(repo_id=DATASET_REPO, repo_type="dataset", filename="books.parquet") df = pd.read_parquet(path) rng = random.Random(42) df["dedup"] = [rng.randint(60, 94) for _ in range(len(df))] return df except Exception as e: print(f"books load failed: {e}") return pd.DataFrame(columns=["isbn13","title","author","year_published","goodreads_shelf", "community_rating_count","community_rating_sum", "genre_tags","mood_tags","dedup"]) def load_contributions() -> pd.DataFrame: try: path = hf_hub_download(repo_id=DATASET_REPO, repo_type="dataset", filename="contributions.parquet") return pd.read_parquet(path) except Exception as e: print(f"contributions load failed: {e}") return pd.DataFrame(columns=["contribution_id","isbn13","hf_username","shelf", "rating","genre_tags","mood_tags","contributed_at"]) def merge_contribs(books: pd.DataFrame, contribs: pd.DataFrame) -> pd.DataFrame: if contribs.empty: return books rated = (contribs[contribs["rating"] > 0] .groupby("isbn13") .agg(rc=("rating","count"), rs=("rating","sum")) .reset_index()) books = books.merge(rated, on="isbn13", how="left") orig_rc = books.get("community_rating_count", pd.Series(0, index=books.index)) orig_rs = books.get("community_rating_sum", pd.Series(0, index=books.index)) books["community_rating_count"] = books["rc"].fillna(orig_rc).fillna(0).astype(int) books["community_rating_sum"] = books["rs"].fillna(orig_rs).fillna(0).astype(int) books.drop(columns=["rc","rs"], errors="ignore", inplace=True) return books def build_leaderboard(contribs: pd.DataFrame) -> list: if contribs.empty: return [] lb = (contribs.groupby("hf_username") .agg(contributions=("contribution_id","count"), joined=("contributed_at","min")) .reset_index() .sort_values("contributions", ascending=False) .head(50)) return lb.to_dict("records") def get_data(refresh=False): if refresh or "books" not in _cache: books = load_books() contribs = load_contributions() _cache["books"] = merge_contribs(books, contribs) _cache["contribs"] = contribs _cache["leaders"] = build_leaderboard(contribs) return _cache["books"], _cache["contribs"], _cache["leaders"] CHUNK_SIZE = 1024 # 1 KB chunks; Xet uses 64 KB CDC at scale def compute_dedup(old_bytes: bytes, new_bytes: bytes) -> dict: def chunk_hashes(data): return [hashlib.sha256(data[i:i+CHUNK_SIZE]).hexdigest() for i in range(0, len(data), CHUNK_SIZE)] old_set = set(chunk_hashes(old_bytes)) if old_bytes else set() new_list = chunk_hashes(new_bytes) shared = sum(1 for h in new_list if h in old_set) total = len(new_list) or 1 dedup_pct = round(shared / total * 100) total_kb = max(1, len(new_bytes) // 1024) saved_kb = round(total_kb * dedup_pct / 100) return { "dedup_pct": dedup_pct, "total_kb": total_kb, "saved_kb": saved_kb, "new_kb": max(1, total_kb - saved_kb), "total_chunks": total, "new_chunks": total - shared, } def append_contribution(contrib: dict) -> dict: _, contribs, _ = get_data() # Serialize old state for dedup comparison old_buf = io.BytesIO() contribs.to_parquet(old_buf, compression="snappy", index=False) old_bytes = old_buf.getvalue() # Build and serialize new state updated = pd.concat([contribs, pd.DataFrame([contrib])], ignore_index=True) new_buf = io.BytesIO() updated.to_parquet(new_buf, compression="snappy", index=False) new_bytes = new_buf.getvalue() # Real chunk-level dedup stats stats = compute_dedup(old_bytes, new_bytes) HfApi(token=WRITE_TOKEN).upload_file( path_or_fileobj=io.BytesIO(new_bytes), path_in_repo="contributions.parquet", repo_id=DATASET_REPO, repo_type="dataset", commit_message=f"Add contribution by {contrib['hf_username']}", ) _cache.clear() return stats # ── HTML helpers ────────────────────────────────────────────────────────────── def _tag(text, kind): style = ("background:rgba(193,68,14,0.08);color:#C1440E" if kind == "mood" else "background:rgba(92,122,95,0.1);color:#5C7A5F") return (f'{text}') def _stars(n): return '' + "".join("★" if i < n else "☆" for i in range(5)) + "" # ── Page sections ───────────────────────────────────────────────────────────── HEADER_HTML = """
The Open Shelf
Community Book Dataset
huggingface.co/datasets/open-shelf →
""" FOOTER_HTML = f"""
The Open Shelf · Community Book Dataset · huggingface.co/datasets/open-shelf Powered by Xet Storage · Built on 🤗 Hub
""" LOGIN_GATE_HTML = """
Sign in to contribute

Connect your Hugging Face account to rate books, add new titles, and appear on the leaderboard.

""" def render_stats_bar(books, contribs, leaders): n_books = len(books) n_ratings = int((contribs["rating"] > 0).sum()) if not contribs.empty else 0 n_contrib = len(leaders) return f"""
{n_books:,} books
{n_ratings:,} ratings
{n_contrib:,} contributors
saved by XET 1.12 GB deduped
""" def render_home_books(books): rated = books[books["community_rating_count"] > 0].sort_values("community_rating_count", ascending=False).head(3) if rated.empty: rated = books.head(3) cards = "" for _, row in rated.iterrows(): avg = round(row["community_rating_sum"] / row["community_rating_count"]) if row.get("community_rating_count", 0) > 0 else 0 genres = [g for g in str(row.get("genre_tags","")).split("|") if g][:1] moods = [m for m in str(row.get("mood_tags","")).split("|") if m][:1] tags = "".join(_tag(g,"genre") for g in genres) + "".join(_tag(m,"mood") for m in moods) cards += f"""
{row.get("title","")}
{row.get("author","")}
{tags}
{_stars(avg)}
{int(row.get("community_rating_count",0))} ratings · ♻ {row.get("dedup",70)}% deduped
""" return f'
{cards}
' def render_home_leaders(leaders): if not leaders: return "

No contributors yet.

" rows = "" for i, l in enumerate(leaders[:4]): color = AVATAR_COLORS[i % len(AVATAR_COLORS)] joined = str(l.get("joined",""))[:7] gold = "color:#B8860B;font-weight:600" if i == 0 else "color:#78716C" rows += f"""
#{i+1}
{l["hf_username"][0].upper()}
{l["hf_username"]}
since {joined}
{l["contributions"]}contributions
""" return f'
{rows}
' def render_browse_table(books, search="", shelf="all"): df = books.copy() if search.strip(): q = search.strip().lower() df = df[df["title"].str.lower().str.contains(q, na=False) | df["author"].str.lower().str.contains(q, na=False)] if shelf != "all": df = df[df["goodreads_shelf"] == shelf] total = len(df) df = df.head(200) rows = "" for _, row in df.iterrows(): avg = round(row["community_rating_sum"] / row["community_rating_count"]) if row.get("community_rating_count", 0) > 0 else 0 genres = [g for g in str(row.get("genre_tags","")).split("|") if g][:1] moods = [m for m in str(row.get("mood_tags","")).split("|") if m][:1] tags = "".join(_tag(g,"genre") for g in genres) + "".join(_tag(m,"mood") for m in moods) year = int(row["year_published"]) if row.get("year_published") and int(row.get("year_published",0)) > 0 else "—" rows += f""" {row.get("title","")} {row.get("author","")} {year} {tags} {_stars(avg)} {int(row.get("community_rating_count",0))} {row.get("dedup",70)}% """ note = (f'

' f'Showing 200 of {total:,} — refine search to see more

') if total > 200 else "" return f"""
{total:,} books
{"".join(f'' for h in ["Title","Author","Year","Tags","Rating","Ratings","♻ dedup"])} {rows}
{h}
{note}""" def render_leaderboard(leaders): if not leaders: return "

No contributors yet — be the first!

" max_c = leaders[0]["contributions"] rows = "" for i, l in enumerate(leaders): color = AVATAR_COLORS[i % len(AVATAR_COLORS)] joined = str(l.get("joined",""))[:7] pct = int(l["contributions"] / max_c * 100) top3 = "color:#B8860B" if i < 3 else "color:#EAE3D2" rows += f"""
{i+1}
{l["hf_username"][0].upper()}
{l["hf_username"]}
contributor since {joined}
{l["contributions"]}
contributions
""" return rows def render_receipt(title, author, shelf, rating, genres, moods, stats: dict): dedup_pct = stats["dedup_pct"] total_kb = stats["total_kb"] saved_kb = stats["saved_kb"] new_kb = stats["new_kb"] total_chunks= stats["total_chunks"] new_chunks = stats["new_chunks"] tags = "" if shelf: tags += _tag(shelf, "genre") if rating: tags += _tag("★" * rating, "mood") for g in genres: tags += _tag(g, "genre") for m in moods: tags += _tag(m, "mood") return f"""
✓ Contribution saved to Hub dataset
{title}
{author}
{tags}
♻ Storage breakdown
Updated dataset size{total_kb} KB · {total_chunks} chunks
Already in Xet store−{saved_kb} KB ({dedup_pct}% dedup)
Actually uploaded{new_kb} KB · {new_chunks} new chunks
Computed via 1 KB chunk hashing on the real parquet bytes.
Xet uses 64 KB CDC — dedup accelerates as the dataset reaches GB scale.
XET chunk-level dedup · committed to huggingface.co/datasets/open-shelf
""" # ── Gradio app ───────────────────────────────────────────────────────────────── CSS = """ @import url('https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,700;0,900;1,700&family=Crimson+Pro:ital,wght@0,400;0,600;1,400&family=JetBrains+Mono:wght@400;600&display=swap'); body, .gradio-container { background:#F5F0E8 !important; font-family:'Crimson Pro',Georgia,serif !important; } .gradio-container { max-width:100% !important; padding:0 !important; } footer.svelte-1rjryqp, footer { display:none !important; } .tab-nav { border-bottom:2px solid #1C1917 !important; background:#FDFAF4 !important; padding:0 2.5rem !important; gap:0 !important; } .tab-nav button { font-family:'Crimson Pro',serif !important; font-size:1rem !important; font-weight:600 !important; color:#78716C !important; border:none !important; border-bottom:3px solid transparent !important; padding:0.85rem 1.25rem !important; background:none !important; border-radius:0 !important; margin:0 !important; } .tab-nav button.selected { color:#C1440E !important; border-bottom-color:#C1440E !important; } .main-pad { max-width:1200px; margin:0 auto; padding:2.5rem; } """ with gr.Blocks(css=CSS, title="The Open Shelf") as demo: gr.HTML(HEADER_HTML) stats_bar = gr.HTML() with gr.Tabs(): # ── Home ───────────────────────────────────────────────── with gr.Tab("🏠 Home"): with gr.Column(elem_classes="main-pad"): gr.HTML("""
Community Dataset · Hugging Face Hub

Every book
a shared
discovery

An open, community-built reading dataset. Browse thousands of books, rate and tag what you've read — and watch the collection grow without wasting a byte.

Recently Rated
""") home_books_html = gr.HTML() gr.HTML('
Top Readers
') home_leaders_html = gr.HTML() gr.HTML("
") # ── Browse ─────────────────────────────────────────────── with gr.Tab("📚 Browse"): with gr.Column(elem_classes="main-pad"): gr.HTML('

Browse Books

') with gr.Row(): search_input = gr.Textbox(placeholder="Search by title or author…", show_label=False, scale=3) shelf_filter = gr.Dropdown(choices=["all","read","to-read","currently-reading"], value="all", label="Shelf", scale=1) browse_html = gr.HTML() def do_browse(query, shelf): books, _, _ = get_data() return render_browse_table(books, search=query or "", shelf=shelf or "all") search_input.change(do_browse, [search_input, shelf_filter], browse_html) shelf_filter.change(do_browse, [search_input, shelf_filter], browse_html) # ── Contribute ─────────────────────────────────────────── with gr.Tab("✏️ Contribute"): with gr.Column(elem_classes="main-pad"): gr.HTML('

Contribute

') login_gate = gr.HTML(LOGIN_GATE_HTML) login_btn = gr.LoginButton(value="🤗 Sign in with Hugging Face") with gr.Group(visible=False) as contribute_form: with gr.Row(): with gr.Column(scale=2): book_search = gr.Textbox(label="Search for a book", placeholder="Type a title or author…") book_choice = gr.Dropdown(label="Select book", choices=[], interactive=True, visible=False) shelf_input = gr.Radio(choices=["read","to-read","currently-reading"], value="read", label="Your Shelf") rating_input = gr.Slider(minimum=0, maximum=5, step=1, value=0, label="Your Rating (0 = unrated)") genre_input = gr.CheckboxGroup(choices=GENRE_TAGS, label="Genre Tags") mood_input = gr.CheckboxGroup(choices=MOOD_TAGS, label="Mood Tags") submit_btn = gr.Button("Submit Contribution →", variant="primary") with gr.Column(scale=1): gr.HTML("""
♻ Your contribution, stored efficiently
Every submission writes a small incremental commit via Xet. Chunks already present are never re-uploaded — as the community grows, each contribution costs less storage than the last.
📊 Your impact
Contributions are attributed to your HF username and appear on the leaderboard. Your data stays open for researchers building recommendation systems and reading analytics.
""") receipt_html = gr.HTML(visible=False) status_html = gr.HTML() def update_book_choices(query): books, _, _ = get_data() if not query or len(query) < 2: return gr.update(choices=[], visible=False) q = query.lower() matches = books[ books["title"].str.lower().str.contains(q, na=False) | books["author"].str.lower().str.contains(q, na=False) ].head(15) choices = [f"{r['title']} — {r['author']}" for _, r in matches.iterrows()] return gr.update(choices=choices, visible=bool(choices), value=choices[0] if choices else None) book_search.change(update_book_choices, book_search, book_choice) def handle_submit(book_sel, shelf, rating, genres, moods, oauth_profile: gr.OAuthProfile | None): if oauth_profile is None: return gr.update(visible=False), gr.update(value="

Please sign in first.

") if not book_sel: return gr.update(visible=False), gr.update(value="

Please select a book.

") if not WRITE_TOKEN: return gr.update(visible=False), gr.update(value="

HF_WRITE_TOKEN secret not configured.

") books, _, _ = get_data() title_part = book_sel.split(" — ")[0].strip() match = books[books["title"] == title_part] if match.empty: match = books[books["title"].str.contains(title_part, na=False)] row = match.iloc[0] if not match.empty else None isbn = str(row["isbn13"]) if row is not None else "" title = str(row["title"]) if row is not None else title_part author = str(row["author"]) if row is not None else "" contrib = { "contribution_id": str(uuid.uuid4()), "isbn13": isbn, "hf_username": oauth_profile.username, "shelf": shelf, "rating": int(rating), "genre_tags": "|".join(genres), "mood_tags": "|".join(moods), "contributed_at": datetime.now(timezone.utc).isoformat(), } try: stats = append_contribution(contrib) except Exception as e: return gr.update(visible=False), gr.update(value=f"

Write failed: {e}

") receipt = render_receipt(title, author, shelf, int(rating), genres, moods, stats) return gr.update(value=receipt, visible=True), gr.update(value="") submit_btn.click(handle_submit, inputs=[book_choice, shelf_input, rating_input, genre_input, mood_input], outputs=[receipt_html, status_html]) # ── Leaderboard ────────────────────────────────────────── with gr.Tab("🏆 Leaderboard"): with gr.Column(elem_classes="main-pad"): gr.HTML('

Top Contributors

') leaderboard_html = gr.HTML() gr.HTML(FOOTER_HTML) # ── Load events ─────────────────────────────────────────────── def load_all_data(profile: gr.OAuthProfile | None): books, contribs, leaders = get_data() logged_in = profile is not None return ( render_stats_bar(books, contribs, leaders), render_home_books(books), render_home_leaders(leaders), render_browse_table(books), render_leaderboard(leaders), gr.update(visible=not logged_in), gr.update(visible=logged_in), ) demo.load(load_all_data, outputs=[stats_bar, home_books_html, home_leaders_html, browse_html, leaderboard_html, login_gate, contribute_form]) login_btn.click(load_all_data, outputs=[stats_bar, home_books_html, home_leaders_html, browse_html, leaderboard_html, login_gate, contribute_form]) if __name__ == "__main__": demo.launch()