Use HF dataset logs and improve survey outputs

.env.example

@@ -1,5 +1,6 @@
 OPENAI_API_KEY=
 SILICON_LLM_MODEL=gpt-4o-mini
 SILICON_LLM_MAX_AGENTS=3000
-GOOGLE_SERVICE_ACCOUNT_JSON=
-GOOGLE_DRIVE_FOLDER_ID=
+HF_LOG_REPO_ID=kyu823/silicon-sampling-logs
+HF_LOG_TOKEN=
+HF_LOG_REPO_PRIVATE=true

README.md

@@ -37,9 +37,26 @@ http://127.0.0.1:8765/silicon/
 
 ## Hugging Face Spaces
 
-This repo is configured as a Docker Space. Set `OPENAI_API_KEY` as a Space secret, not as a committed file.
+This repo is configured as a Docker Space. For Hugging Face Spaces, set secrets in the Space dashboard under Settings → Secrets, not in a committed file.
 
-The app listens on port `7860`.
+Required secrets:
+
+- `OPENAI_API_KEY`: your OpenAI API key. This is required for the LLM-backed simulation runs.
+
+Optional secrets:
+
+- `HF_LOG_REPO_ID`: Hugging Face Dataset repo for run logs. Defaults to `kyu823/silicon-sampling-logs`.
+- `HF_LOG_TOKEN`: Hugging Face token with write access to the log dataset repo. If omitted, the backend falls back to `HF_TOKEN`.
+- `HF_LOG_REPO_PRIVATE`: set to `true` to create/use a private log dataset repo.
+
+Deployment flow:
+
+1. Create or connect a Docker Space for this repository.
+2. Add the above secrets in the Space settings.
+3. Deploy the Space.
+4. The app listens on port `7860`.
+
+The backend reads these values from environment variables, so Space secrets are the correct way to provide them. Without `OPENAI_API_KEY`, the app can still start, but the simulation run will fail when the LLM call is attempted.
 
 ## Logs
 
@@ -49,14 +66,15 @@ Every completed silicon-sampling run is logged anonymously. By default logs are
 - `runs/silicon_logs.sqlite3` locally
 - `/app/runs/silicon_logs.sqlite3` on a default Hugging Face Docker Space without persistent storage
 
-Google Drive mirroring is optional. To enable it, create a Google Cloud service account with Drive API access, share one Drive folder with the service account email as an editor, then set these Space secrets:
+Hugging Face Dataset mirroring is optional and recommended for deployed runs. To enable it, set:
 
 ```text
-GOOGLE_SERVICE_ACCOUNT_JSON=<raw service account JSON or base64-encoded JSON>
-GOOGLE_DRIVE_FOLDER_ID=<target Drive folder ID>
+HF_LOG_REPO_ID=kyu823/silicon-sampling-logs
+HF_LOG_TOKEN=<write token>
+HF_LOG_REPO_PRIVATE=true
 ```
 
-When enabled, each run also writes one JSON file to that Drive folder. The JSON includes respondent settings, question definitions, and aggregate result summaries; raw LLM responses are not mirrored.
+When enabled, each run also writes one JSON file to the dataset repo under `runs/YYYY-MM-DD/`. The JSON includes respondent settings, question definitions, and aggregate result summaries; raw LLM responses are not mirrored.
 
 ## Optional Nemotron Dataset
 

frontend/src/silicon/SiliconCharts.tsx

@@ -5,7 +5,7 @@ import { init, use } from "echarts/core";
 import { CanvasRenderer } from "echarts/renderers";
 import { AGE_BANDS, GENDERS } from "./data";
 import { resultBreakdown } from "./simulate";
-import type { PersonaDimensionId, SiliconResult, SyntheticRespondent } from "./types";
+import type { PersonaDimensionId, SiliconResult, SurveyQuestion, SyntheticRespondent } from "./types";
 
 use([BarChart, GridComponent, TooltipComponent, CanvasRenderer]);
 
@@ -36,9 +36,10 @@ export function SiliconCharts({ result }: SiliconChartsProps) {
   const selectedStat = result.questionStats.find((stat) => stat.questionId === selectedQuestion?.id);
   const isLikert = selectedQuestion?.kind === "likert";
   const isCategorical = selectedQuestion?.kind === "categorical";
+  const showBreakdown = breakdown !== "overall" && (isLikert || isCategorical);
   const breakdownRows = useMemo(
-    () => isLikert && breakdown !== "overall" ? resultBreakdown(result, selectedQuestion.id, breakdown) : [],
-    [breakdown, isLikert, result, selectedQuestion?.id],
+    () => (showBreakdown ? resultBreakdown(result, selectedQuestion.id, breakdown) : []),
+    [breakdown, result, selectedQuestion?.id, showBreakdown],
   );
 
   useEffect(() => {
@@ -76,7 +77,7 @@ export function SiliconCharts({ result }: SiliconChartsProps) {
             ].filter(Boolean).join(" ")}
             onClick={() => {
               setSelectedQuestionId(question.id);
-              if (question.kind !== "likert") setBreakdown("overall");
+              if (question.kind === "open") setBreakdown("overall");
             }}
           >
             <span>{question.kind === "likert" ? `${question.scale}점 Likert` : question.kind === "categorical" ? "Categorical" : "Open-ended"}</span>
@@ -87,69 +88,75 @@ export function SiliconCharts({ result }: SiliconChartsProps) {
 
       {isLikert ? (
         <>
-          <div className="metric-tabs">
-            <button type="button" className={metricMode === "mean" ? "active" : ""} onClick={() => setMetricMode("mean")}>평균</button>
-            <button type="button" className={metricMode === "positive" ? "active" : ""} onClick={() => setMetricMode("positive")}>긍정률</button>
-          </div>
+          {breakdown !== "overall" ? (
+            <div className="metric-tabs">
+              <button type="button" className={metricMode === "mean" ? "active" : ""} onClick={() => setMetricMode("mean")}>평균</button>
+              <button type="button" className={metricMode === "positive" ? "active" : ""} onClick={() => setMetricMode("positive")}>긍정률</button>
+            </div>
+          ) : null}
           <div className="silicon-chart-grid single">
+            <ChartCard
+              title={breakdown === "overall" ? "응답 분포" : `${modeLabel(breakdown)} ${metricMode === "mean" ? "평균" : "긍정률"}`}
+              subtitle={selectedQuestion.title}
+              option={breakdown === "overall" ? {
+                color: ["#5b6ee1"],
+                tooltip: tooltip(),
+                grid: grid(),
+                xAxis: { type: "category", data: selectedStat?.distribution?.map((item) => `${item.value}`) || [], axisLabel: axisLabel() },
+                yAxis: { type: "value", axisLabel: axisLabel(), splitLine: splitLine() },
+                series: [{
+                  type: "bar",
+                  data: selectedStat?.distribution?.map((item) => item.count) || [],
+                  barWidth: "52%",
+                  itemStyle: { borderRadius: [8, 8, 0, 0] },
+                }],
+              } : likertBreakdownOption(breakdownRows, metricMode)}
+            />
             {breakdown === "overall" ? (
-              <ChartCard
-                title="응답 분포"
-                subtitle={selectedQuestion.title}
-                option={{
-                  color: ["#5b6ee1"],
-                  tooltip: tooltip(),
-                  grid: grid(),
-                  xAxis: { type: "category", data: selectedStat?.distribution?.map((item) => `${item.value}`) || [], axisLabel: axisLabel() },
-                  yAxis: { type: "value", axisLabel: axisLabel(), splitLine: splitLine() },
-                  series: [{
-                    type: "bar",
-                    data: selectedStat?.distribution?.map((item) => item.count) || [],
-                    barWidth: "52%",
-                    itemStyle: { borderRadius: [8, 8, 0, 0] },
-                  }],
-                }}
-              />
-            ) : (
-              <ChartCard
-                title={`${modeLabel(breakdown)} ${metricMode === "mean" ? "평균" : "긍정률"}`}
-                subtitle={selectedQuestion.title}
-                option={breakdownOption(breakdownRows, metricMode)}
+              <MetricPanel
+                mean={selectedStat?.mean}
+                positiveShare={selectedStat?.positiveShare}
+                count={result.likertAnswers.filter((answer) => answer.questionId === selectedQuestion.id).length}
+                scale={selectedQuestion.scale || 5}
               />
-            )}
-            <MetricPanel
-              mean={selectedStat?.mean}
-              positiveShare={selectedStat?.positiveShare}
-              count={result.likertAnswers.filter((answer) => answer.questionId === selectedQuestion.id).length}
-              scale={selectedQuestion.scale || 5}
-            />
+            ) : null}
             <ResponseTable result={result} questionId={selectedQuestion.id} />
           </div>
         </>
       ) : isCategorical ? (
         <div className="silicon-chart-grid single">
-          <ChartCard
-            title="선택지별 응답 분포"
-            subtitle={selectedQuestion.title}
-            option={{
-              color: ["#6f7deb"],
-              tooltip: tooltip((value) => `${Number(value).toLocaleString()}개`),
-              grid: { ...grid(), left: 80 },
-              xAxis: { type: "value", axisLabel: axisLabel(), splitLine: splitLine() },
-              yAxis: { type: "category", data: selectedStat?.distribution?.map((item) => item.label || item.optionId || "") || [], axisLabel: axisLabel() },
-              series: [{
-                type: "bar",
-                data: selectedStat?.distribution?.map((item) => item.count) || [],
-                barWidth: "54%",
-                itemStyle: { borderRadius: [0, 8, 8, 0] },
-              }],
-            }}
-          />
-          <CategoricalMetricPanel
-            count={result.categoricalAnswers.filter((answer) => answer.questionId === selectedQuestion.id).length}
-            options={selectedQuestion.options?.length || 0}
-            topLabel={topCategoricalLabel(selectedStat?.distribution)}
-          />
+          {breakdown === "overall" ? (
+            <ChartCard
+              title="선택지별 응답 분포"
+              subtitle={selectedQuestion.title}
+              option={{
+                color: ["#6f7deb"],
+                tooltip: tooltip((value) => `${Number(value).toLocaleString()}개`),
+                grid: { ...grid(), left: 80 },
+                xAxis: { type: "value", axisLabel: axisLabel(), splitLine: splitLine() },
+                yAxis: { type: "category", data: selectedStat?.distribution?.map((item) => item.label || item.optionId || "") || [], axisLabel: axisLabel() },
+                series: [{
+                  type: "bar",
+                  data: selectedStat?.distribution?.map((item) => item.count) || [],
+                  barWidth: "54%",
+                  itemStyle: { borderRadius: [0, 8, 8, 0] },
+                }],
+              }}
+            />
+          ) : (
+            <ChartCard
+              title="집단별 응답 분포"
+              subtitle={selectedQuestion.title}
+              option={categoricalBreakdownOption(breakdownRows, selectedQuestion)}
+            />
+          )}
+          {breakdown === "overall" ? (
+            <CategoricalMetricPanel
+              count={result.categoricalAnswers.filter((answer) => answer.questionId === selectedQuestion.id).length}
+              options={selectedQuestion.options?.length || 0}
+              topLabel={topCategoricalLabel(selectedStat?.distribution)}
+            />
+          ) : null}
           <ResponseTable result={result} questionId={selectedQuestion.id} />
         </div>
       ) : (
@@ -318,7 +325,7 @@ function topCategoricalLabel(distribution?: Array<{ label?: string; optionId?: s
   return `${top.label || top.optionId || "-"} (${top.count.toLocaleString()}개)`;
 }
 
-function breakdownOption(rows: ReturnType<typeof resultBreakdown>, metricMode: MetricMode) {
+function likertBreakdownOption(rows: ReturnType<typeof resultBreakdown>, metricMode: MetricMode) {
   const isPositive = metricMode === "positive";
   return {
     color: [isPositive ? "#f08a6c" : "#5b6ee1"],
@@ -340,6 +347,40 @@ function breakdownOption(rows: ReturnType<typeof resultBreakdown>, metricMode: M
   };
 }
 
+function categoricalBreakdownOption(rows: ReturnType<typeof resultBreakdown>, question: SurveyQuestion) {
+  const buckets = (question.options || []).map((option) => ({ id: option.id, label: option.label }));
+  const colors = ["#5b6ee1", "#4db7c0", "#f0b56c", "#f08a6c", "#8d7cf3", "#53b56a", "#7e8bd8"];
+  return {
+    tooltip: {
+      trigger: "axis",
+      backgroundColor: "rgba(255, 255, 255, .96)",
+      borderColor: "rgba(84, 96, 137, .18)",
+      textStyle: { color: "#18213a" },
+      axisPointer: { type: "shadow" },
+    },
+    legend: {
+      data: buckets.map((bucket) => bucket.label),
+      top: 0,
+      textStyle: { color: "rgba(24, 33, 58, .7)", fontSize: 11 },
+    },
+    grid: { ...grid(), left: 92, right: 18, top: 46, bottom: 24 },
+    xAxis: {
+      type: "value",
+      axisLabel: axisLabel(),
+      splitLine: splitLine(),
+    },
+    yAxis: { type: "category", data: rows.map((item) => item.label), axisLabel: axisLabel() },
+    series: buckets.map((bucket, index) => ({
+      name: bucket.label,
+      type: "bar",
+      stack: "total",
+      data: rows.map((row) => row.distribution.find((item) => item.id === bucket.id)?.count || 0),
+      barWidth: "54%",
+      itemStyle: { color: colors[index % colors.length], borderRadius: [0, 0, 0, 0] },
+    })),
+  };
+}
+
 function tooltip(formatter?: (value: unknown) => string) {
   return {
     trigger: "item",

frontend/src/silicon/SiliconSamplingApp.tsx

@@ -52,6 +52,7 @@ const FALLBACK_NEMOTRON_COLUMNS = [
 type CountRow = { value: string; count: number };
 type FlowStage = "respondents" | "questions" | "results";
 type FilterValue<T extends string> = typeof ALL_FILTER | T;
+type PromptPreview = { systemContent: string; userContent: string; sampleRespondent?: unknown; questionCount?: number };
 type NemotronMetadata = {
   columns?: string[];
   sex_counts?: CountRow[];
@@ -92,6 +93,8 @@ export function SiliconSamplingApp() {
   const [result, setResult] = useState<SiliconResult | null>(null);
   const [isRunning, setIsRunning] = useState(false);
   const [runStatus, setRunStatus] = useState("응답자 조건을 확인하고 응답자를 생성하세요.");
+  const [promptPreview, setPromptPreview] = useState<PromptPreview | null>(null);
+  const [promptPreviewStatus, setPromptPreviewStatus] = useState("");
 
   const sidoOptions = useMemo(() => {
     const options = locationOptions.filter((location) => location.level === "sido");
@@ -157,16 +160,46 @@ export function SiliconSamplingApp() {
     if (!generatedRespondents.length) return;
     setGeneratedRespondents([]);
     setResult(null);
+    setPromptPreview(null);
     setRunStatus("응답자 조건이 바뀌었습니다. 응답자를 다시 생성하세요.");
   }, [sampleSize, genderFilter, ageFilter, regionFilter]); // eslint-disable-line react-hooks/exhaustive-deps
 
+  useEffect(() => {
+    setPromptPreview(null);
+    setPromptPreviewStatus("");
+  }, [questions]);
+
   const generateRespondents = () => {
     const preview = simulateSiliconSampling({ ...config, questions: [] }).respondents;
     setGeneratedRespondents(preview);
     setResult(null);
+    setPromptPreview(null);
     setRunStatus(`${preview.length.toLocaleString()}명의 응답자 프로필을 생성했습니다. 목록을 확인한 뒤 문항 설정으로 이동하세요.`);
   };
 
+  const loadPromptPreview = async () => {
+    if (!questions.length) {
+      setPromptPreview(null);
+      setPromptPreviewStatus("문항을 하나 이상 선택하면 실행 프롬프트를 확인할 수 있습니다.");
+      return;
+    }
+    setPromptPreviewStatus("실제 백엔드 프롬프트를 불러오는 중입니다.");
+    try {
+      const response = await fetch("/api/silicon/prompt-preview", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ config: { ...config, respondents: generatedRespondents } }),
+      });
+      const payload = await response.json();
+      if (!response.ok) throw new Error(payload?.error || `HTTP ${response.status}`);
+      setPromptPreview(payload as PromptPreview);
+      setPromptPreviewStatus("실제 실행에 사용되는 백엔드 프롬프트입니다.");
+    } catch (error) {
+      setPromptPreview(null);
+      setPromptPreviewStatus(`프롬프트를 불러오지 못했습니다: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  };
+
   const run = async () => {
     if (!canRun) {
       setRunStatus(!generatedRespondents.length ? "먼저 응답자를 생성하세요." : "시뮬레이션 실행 전 문항을 하나 이상 선택하세요.");
@@ -214,6 +247,8 @@ export function SiliconSamplingApp() {
     setQuestionBankOpen(true);
     setGeneratedRespondents([]);
     setResult(null);
+    setPromptPreview(null);
+    setPromptPreviewStatus("");
     setIsRunning(false);
     setRunStatus("초기화했습니다. 기본 비율 조건으로 응답자를 생성하세요.");
   };
@@ -239,6 +274,7 @@ export function SiliconSamplingApp() {
     setCustomLowLabel("");
     setCustomHighLabel("");
     setCustomCategoricalOptions(DEFAULT_CATEGORICAL_OPTIONS);
+    setPromptPreview(null);
   };
 
   useEffect(() => {
@@ -326,6 +362,9 @@ export function SiliconSamplingApp() {
             customCategoricalOptions={customCategoricalOptions}
             setCustomCategoricalOptions={setCustomCategoricalOptions}
             questions={questions}
+            promptPreview={promptPreview}
+            promptPreviewStatus={promptPreviewStatus}
+            onLoadPromptPreview={loadPromptPreview}
             onAddCustomQuestion={addCustomQuestion}
             onBack={() => setStage("respondents")}
             onRun={run}
@@ -472,6 +511,9 @@ function QuestionSetup({
   customCategoricalOptions,
   setCustomCategoricalOptions,
   questions,
+  promptPreview,
+  promptPreviewStatus,
+  onLoadPromptPreview,
   onAddCustomQuestion,
   onBack,
   onRun,
@@ -496,11 +538,22 @@ function QuestionSetup({
   customCategoricalOptions: string[];
   setCustomCategoricalOptions: (value: string[] | ((current: string[]) => string[])) => void;
   questions: SurveyQuestion[];
+  promptPreview: PromptPreview | null;
+  promptPreviewStatus: string;
+  onLoadPromptPreview: () => void;
   onAddCustomQuestion: () => void;
   onBack: () => void;
   onRun: () => void;
   canRun: boolean;
 }) {
+  const [promptPreviewOpen, setPromptPreviewOpen] = useState(false);
+  const togglePromptPreview = () => {
+    setPromptPreviewOpen((value) => {
+      const next = !value;
+      if (next) onLoadPromptPreview();
+      return next;
+    });
+  };
   const removeQuestion = (question: SurveyQuestion) => {
     if (question.id.startsWith("custom_")) {
       setCustomQuestions((items) => items.filter((item) => item.id !== question.id));
@@ -614,6 +667,35 @@ function QuestionSetup({
         </div>
       </section>
 
+      <section className="prompt-preview-card">
+        <button
+          type="button"
+          className={promptPreviewOpen ? "question-bank-toggle open" : "question-bank-toggle"}
+          aria-expanded={promptPreviewOpen}
+          onClick={togglePromptPreview}
+        >
+          <ChevronDown size={15} />
+          실행 전 프롬프트 구성 미리보기 {promptPreviewOpen ? "접기" : "열기"}
+        </button>
+        {promptPreviewOpen ? (
+          <div className="prompt-preview-body">
+            {promptPreviewStatus ? <p className="prompt-preview-status">{promptPreviewStatus}</p> : null}
+            {promptPreview ? (
+              <>
+                <div className="prompt-preview-block">
+                  <span>system</span>
+                  <pre>{promptPreview.systemContent}</pre>
+                </div>
+                <div className="prompt-preview-block">
+                  <span>user</span>
+                  <pre>{promptPreview.userContent}</pre>
+                </div>
+              </>
+            ) : null}
+          </div>
+        ) : null}
+      </section>
+
       <section className="final-question-card" data-testid="silicon-final-question-list">
         <header>
           <div>

frontend/src/silicon/simulate.ts

@@ -294,9 +294,17 @@ export function groupBreakdown(result: SiliconResult, dimension: "gender" | "age
 
 export function resultBreakdown(result: SiliconResult, questionId: string, dimension: "gender" | "age" | "region" | PersonaDimensionId) {
   const question = result.config.questions.find((item) => item.id === questionId);
-  if (!question || question.kind !== "likert") return [];
-  const answers = result.likertAnswers.filter((answer) => answer.questionId === questionId);
-  const valuesByRespondent = new Map(answers.map((answer) => [answer.respondentId, answer.value]));
+  if (!question || (question.kind !== "likert" && question.kind !== "categorical")) return [];
+  const valuesByRespondent = new Map<string, number | string>();
+  if (question.kind === "likert") {
+    for (const answer of result.likertAnswers.filter((item) => item.questionId === questionId)) {
+      valuesByRespondent.set(answer.respondentId, answer.value);
+    }
+  } else {
+    for (const answer of result.categoricalAnswers.filter((item) => item.questionId === questionId)) {
+      valuesByRespondent.set(answer.respondentId, answer.optionId);
+    }
+  }
   const scale = question.scale || 5;
   const positiveCut = Math.max(3, Math.ceil(scale * 0.7));
   const options = dimension === "gender"
@@ -315,13 +323,25 @@ export function resultBreakdown(result: SiliconResult, questionId: string, dimen
       if (dimension === "region") return respondent.location === id;
       return respondent.personaAttributes[dimension] === id;
     });
-    const values = people.map((respondent) => valuesByRespondent.get(respondent.id)).filter((value): value is number => typeof value === "number");
+    const values = people.map((respondent) => valuesByRespondent.get(respondent.id)).filter((value): value is number | string => value !== undefined);
+    const numericValues = values.filter((value): value is number => typeof value === "number");
+    const distribution = question.kind === "likert"
+      ? Array.from({ length: scale }, (_, index) => {
+          const value = index + 1;
+          const count = values.filter((item) => item === value).length;
+          return { id: String(value), label: `${value}점`, count, share: values.length ? count / values.length : 0 };
+        })
+      : (question.options || []).map((questionOption) => {
+          const count = values.filter((item) => item === questionOption.id).length;
+          return { id: questionOption.id, label: questionOption.label, count, share: values.length ? count / values.length : 0 };
+        });
     return {
       id,
       label,
       respondents: people.length,
-      mean: mean(values),
-      positiveShare: values.length ? values.filter((value) => value >= positiveCut).length / values.length : 0,
+      mean: mean(numericValues),
+      positiveShare: numericValues.length ? numericValues.filter((value) => value >= positiveCut).length / numericValues.length : 0,
+      distribution,
     };
   }).filter((item) => item.respondents > 0);
 }

frontend/src/styles.css

@@ -1533,6 +1533,60 @@ button {
   margin-top: 14px;
 }
 
+.prompt-preview-card {
+  margin-top: 14px;
+  border: 1px solid rgba(84, 96, 137, 0.14);
+  border-radius: 22px;
+  padding: 18px;
+  background: rgba(255, 255, 255, 0.92);
+  box-shadow: 0 18px 52px rgba(39, 55, 96, 0.09);
+}
+
+.prompt-preview-card .question-bank-toggle {
+  width: 100%;
+  justify-content: flex-start;
+  margin-left: 0;
+}
+
+.prompt-preview-body {
+  display: grid;
+  gap: 12px;
+  margin-top: 12px;
+}
+
+.prompt-preview-status {
+  margin: 0;
+  color: rgba(24, 33, 58, 0.62);
+  font-size: 13px;
+  line-height: 1.5;
+}
+
+.prompt-preview-block {
+  border: 1px solid rgba(84, 96, 137, 0.13);
+  border-radius: 16px;
+  padding: 12px;
+  background: #fbfcff;
+}
+
+.prompt-preview-block span {
+  display: block;
+  color: rgba(24, 33, 58, 0.58);
+  font-size: 11px;
+  font-weight: 850;
+  letter-spacing: 0.07em;
+  text-transform: uppercase;
+}
+
+.prompt-preview-block pre {
+  margin: 8px 0 0;
+  white-space: pre-wrap;
+  word-break: break-word;
+  font-family: Consolas, "Courier New", monospace;
+  font-size: 12px;
+  line-height: 1.6;
+  color: #18213a;
+}
+
 .respondent-preview-card header,
 .final-question-card header {
   display: flex;

hf_dataset_logs.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import io
+import json
+import os
+from typing import Any
+
+
+DEFAULT_LOG_REPO_ID = "kyu823/silicon-sampling-logs"
+
+
+def hf_log_status() -> dict[str, Any]:
+    repo_id = hf_log_repo_id()
+    token = hf_log_token()
+    return {
+        "enabled": bool(repo_id and token),
+        "repoId": repo_id,
+        "tokenSet": bool(token),
+        "private": hf_log_private(),
+    }
+
+
+def upload_run_log_to_hf_dataset(run_document: dict[str, Any]) -> dict[str, Any]:
+    repo_id = hf_log_repo_id()
+    token = hf_log_token()
+    if not repo_id:
+        raise RuntimeError("HF_LOG_REPO_ID is required for Hugging Face Dataset logging")
+    if not token:
+        raise RuntimeError("HF_LOG_TOKEN or HF_TOKEN is required for Hugging Face Dataset logging")
+
+    from huggingface_hub import HfApi
+
+    api = HfApi(token=token)
+    api.create_repo(repo_id, repo_type="dataset", private=hf_log_private(), exist_ok=True)
+    path_in_repo = hf_log_path(run_document)
+    data = json.dumps(run_document, ensure_ascii=False, indent=2, sort_keys=True).encode("utf-8")
+    commit = api.upload_file(
+        path_or_fileobj=io.BytesIO(data),
+        path_in_repo=path_in_repo,
+        repo_id=repo_id,
+        repo_type="dataset",
+        commit_message=f"Add silicon sampling log {run_document.get('runId', 'run')}",
+    )
+    return {
+        "enabled": True,
+        "uploaded": True,
+        "repoId": repo_id,
+        "path": path_in_repo,
+        "commitUrl": getattr(commit, "commit_url", None),
+    }
+
+
+def hf_log_repo_id() -> str:
+    return os.getenv("HF_LOG_REPO_ID", DEFAULT_LOG_REPO_ID).strip()
+
+
+def hf_log_token() -> str:
+    return (os.getenv("HF_LOG_TOKEN") or os.getenv("HF_TOKEN") or "").strip()
+
+
+def hf_log_private() -> bool:
+    return os.getenv("HF_LOG_REPO_PRIVATE", "true").strip().lower() not in {"0", "false", "no"}
+
+
+def hf_log_path(run_document: dict[str, Any]) -> str:
+    created = str(run_document.get("createdAt") or "")
+    day = created[:10] if len(created) >= 10 else "unknown-date"
+    safe_created = created.replace(":", "-").replace("+", "Z")
+    run_id = str(run_document.get("runId") or "run")
+    return f"runs/{day}/{safe_created}_{run_id}.json"

requirements.txt

@@ -1,4 +1,3 @@
 openai>=1.0.0
 duckdb>=1.0.0
-google-api-python-client>=2.0.0
-google-auth>=2.0.0
+huggingface_hub>=0.34.0

scripts/test_silicon_llm_runtime.py

@@ -14,6 +14,8 @@ ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT))
 
 from silicon_llm import (  # noqa: E402
+    build_silicon_llm_system_prompt,
+    build_silicon_prompt_preview,
     build_silicon_llm_response_contract,
     parse_silicon_agent_response,
     run_silicon_llm_sampling,
@@ -86,6 +88,27 @@ class SiliconLLMRuntimeTests(unittest.TestCase):
         self.assertIn("rationale", answer_properties["q_open"]["required"])
         self.assertEqual(answer_properties["q_categorical"]["properties"]["optionId"]["enum"], ["money", "honor", "health"])
 
+    def test_prompt_preview_uses_personalized_backend_prompt(self) -> None:
+        system_prompt = build_silicon_llm_system_prompt()
+        self.assertIn("1인칭", system_prompt)
+        self.assertIn("최소 두 가지", system_prompt)
+        self.assertIn("일반론", system_prompt)
+        preview = build_silicon_prompt_preview(
+            {
+                "config": {
+                    "sampleSize": 1,
+                    "genders": [{"id": "female", "enabled": True, "weight": 1}],
+                    "ages": [{"id": "40s", "enabled": True, "weight": 1}],
+                    "locations": [{"id": "seoul", "enabled": True, "weight": 1}],
+                    "locationOptions": [{"id": "seoul", "label": "서울", "parentRegion": "seoul", "level": "sido"}],
+                    "questions": QUESTIONS,
+                }
+            }
+        )
+        self.assertEqual(preview["systemContent"], system_prompt)
+        self.assertIn("required_json_contract", preview["userContent"])
+        self.assertIn("q_categorical", preview["userContent"])
+
     def test_parser_extracts_all_dynamic_answers_from_fenced_json(self) -> None:
         raw = """
         생각 과정은 생략합니다.

scripts/test_silicon_logs.py

@@ -8,14 +8,13 @@ import sqlite3
 import sys
 import tempfile
 import unittest
-import base64
 from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT))
 
-from google_drive_logs import drive_log_filename, parse_service_account_json  # noqa: E402
+from hf_dataset_logs import hf_log_path  # noqa: E402
 from silicon_logs import log_status, record_run, recent_runs  # noqa: E402
 
 
@@ -24,7 +23,11 @@ class SiliconLogTests(unittest.TestCase):
         with tempfile.TemporaryDirectory() as tmp:
             path = Path(tmp) / "logs.sqlite3"
             old = os.environ.get("SILICON_LOG_DB")
+            old_hf_token = os.environ.get("HF_TOKEN")
+            old_hf_log_token = os.environ.get("HF_LOG_TOKEN")
             os.environ["SILICON_LOG_DB"] = str(path)
+            os.environ.pop("HF_TOKEN", None)
+            os.environ.pop("HF_LOG_TOKEN", None)
             try:
                 info = record_run(
                     {
@@ -54,10 +57,10 @@ class SiliconLogTests(unittest.TestCase):
                     client={"userAgent": "unit-test"},
                 )
                 self.assertTrue(info["runId"].startswith("run_"))
-                self.assertEqual(info["googleDrive"], {"enabled": False, "uploaded": False})
+                self.assertEqual(info["hfDataset"], {"enabled": False, "uploaded": False})
                 self.assertTrue(path.exists())
                 self.assertEqual(log_status()["runCount"], 1)
-                self.assertIn("googleDrive", log_status())
+                self.assertIn("hfDataset", log_status())
                 rows = recent_runs()
                 self.assertEqual(len(rows), 1)
                 self.assertEqual(rows[0]["sampleSize"], 2)
@@ -71,8 +74,12 @@ class SiliconLogTests(unittest.TestCase):
                     os.environ.pop("SILICON_LOG_DB", None)
                 else:
                     os.environ["SILICON_LOG_DB"] = old
+                if old_hf_token is not None:
+                    os.environ["HF_TOKEN"] = old_hf_token
+                if old_hf_log_token is not None:
+                    os.environ["HF_LOG_TOKEN"] = old_hf_log_token
 
-    def test_record_run_can_mirror_json_document_to_google_drive_uploader(self) -> None:
+    def test_record_run_can_mirror_json_document_to_hf_dataset_uploader(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             path = Path(tmp) / "logs.sqlite3"
             old = os.environ.get("SILICON_LOG_DB")
@@ -81,7 +88,7 @@ class SiliconLogTests(unittest.TestCase):
 
             def fake_uploader(document: dict) -> dict:
                 uploaded.append(document)
-                return {"enabled": True, "uploaded": True, "fileId": "drive_file_123", "name": drive_log_filename(document)}
+                return {"enabled": True, "uploaded": True, "repoId": "kyu823/silicon-sampling-logs", "path": hf_log_path(document)}
 
             try:
                 info = record_run(
@@ -104,9 +111,10 @@ class SiliconLogTests(unittest.TestCase):
                         "regionStats": [],
                         "llmTrace": {"calls": 1, "rawResponses": ["not mirrored"]},
                     },
-                    drive_uploader=fake_uploader,
+                    hf_uploader=fake_uploader,
                 )
-                self.assertEqual(info["googleDrive"]["fileId"], "drive_file_123")
+                self.assertEqual(info["hfDataset"]["repoId"], "kyu823/silicon-sampling-logs")
+                self.assertTrue(info["hfDataset"]["path"].startswith("runs/"))
                 self.assertEqual(len(uploaded), 1)
                 self.assertEqual(uploaded[0]["questions"][0]["kind"], "categorical")
                 self.assertEqual(uploaded[0]["respondentSettings"]["locations"][0]["label"], "부산")
@@ -117,12 +125,6 @@ class SiliconLogTests(unittest.TestCase):
                 else:
                     os.environ["SILICON_LOG_DB"] = old
 
-    def test_service_account_json_can_be_plain_or_base64(self) -> None:
-        raw = '{"type":"service_account","client_email":"bot@example.iam.gserviceaccount.com"}'
-        self.assertEqual(parse_service_account_json(raw)["client_email"], "bot@example.iam.gserviceaccount.com")
-        encoded = base64.b64encode(raw.encode("utf-8")).decode("utf-8")
-        self.assertEqual(parse_service_account_json(encoded)["client_email"], "bot@example.iam.gserviceaccount.com")
-
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

server.py

@@ -9,7 +9,7 @@ from urllib.parse import parse_qs, urlparse
 
 from persona_dataset import PersonaDatasetError, dataset_metadata, dataset_status, sample_personas
 from silicon_logs import log_status, recent_runs, record_run
-from silicon_llm import resolve_openai_api_key, run_silicon_llm_sampling
+from silicon_llm import build_silicon_prompt_preview, resolve_openai_api_key, run_silicon_llm_sampling
 
 
 APP_DIR = Path(__file__).resolve().parent
@@ -105,6 +105,11 @@ class Handler(BaseHTTPRequestHandler):
                 return write_json(self, sample_personas(payload))
             except PersonaDatasetError as exc:
                 return write_json(self, {"error": str(exc), "status": dataset_status()}, HTTPStatus.BAD_REQUEST)
+        if parsed.path == "/api/silicon/prompt-preview":
+            try:
+                return write_json(self, build_silicon_prompt_preview(payload))
+            except ValueError as exc:
+                return write_json(self, {"error": str(exc)}, HTTPStatus.BAD_REQUEST)
         if parsed.path == "/api/silicon/llm-run":
             try:
                 result = run_silicon_llm_sampling(payload)

silicon_llm.py

@@ -54,6 +54,39 @@ PERSONA_DIMENSIONS = {
 }
 
 
+def build_silicon_llm_system_prompt() -> str:
+    return (
+        "당신은 한국 설문조사의 가상 응답자입니다. "
+        "주어진 persona와 인구통계 정보를 일관되게 반영해 모든 문항에 답하십시오. "
+        "반드시 JSON만 반환하십시오. Likert 문항은 해당 척도 범위의 정수 value를, "
+        "categorical 문항은 제공된 options 중 하나의 optionId와 label을, "
+        "open-ended 문항은 한국어 text와 짧은 theme를 반환하십시오. "
+        "모든 답변에는 rationale을 포함하십시오. rationale은 응답자의 1인칭 관점으로 쓰고, "
+        "성별, 연령, 지역, 직업, 학력, 주거, 혼인, 가구 등 제공된 배경 중 최소 두 가지를 구체적으로 언급하십시오. "
+        "rationale은 '나는 ... 배경/상황이라서 ...라고 판단했다'처럼 개인 맥락이 보이게 작성하고, "
+        "문항을 반복하거나 일반론만 말하지 마십시오."
+    )
+
+
+def build_silicon_llm_user_prompt(agent: dict[str, Any], questions: list[dict[str, Any]], response_contract: dict[str, Any], *, pretty: bool = False) -> str:
+    return json.dumps(
+        {
+            "agent": agent,
+            "questions": questions,
+            "required_json_contract": response_contract,
+        },
+        ensure_ascii=False,
+        indent=2 if pretty else None,
+    )
+
+
+def build_silicon_llm_messages(agent: dict[str, Any], questions: list[dict[str, Any]], response_contract: dict[str, Any]) -> list[dict[str, str]]:
+    return [
+        {"role": "system", "content": build_silicon_llm_system_prompt()},
+        {"role": "user", "content": build_silicon_llm_user_prompt(agent, questions, response_contract)},
+    ]
+
+
 @dataclass
 class OpenAILLM:
     model: str = "gpt-4o-mini"
@@ -67,30 +100,7 @@ class OpenAILLM:
         if not api_key:
             raise RuntimeError("OPENAI_API_KEY or OPENAI_API_KEY_FILE is required for silicon LLM sampling")
         client = OpenAI(api_key=api_key)
-        messages = [
-            {
-                "role": "system",
-                "content": (
-                    "당신은 한국 설문조사의 가상 응답자입니다. "
-                    "주어진 persona와 인구통계 정보를 일관되게 반영해 모든 문항에 답하십시오. "
-                    "반드시 JSON만 반환하십시오. Likert 문항은 해당 척도 범위의 정수 value를, "
-                    "categorical 문항은 제공된 options 중 하나의 optionId와 label을, "
-                    "open-ended 문항은 한국어 text와 짧은 theme를 반환하십시오. "
-                    "모든 답변에는 왜 그렇게 답했는지 한 문장의 rationale을 포함하십시오."
-                ),
-            },
-            {
-                "role": "user",
-                "content": json.dumps(
-                    {
-                        "agent": agent,
-                        "questions": questions,
-                        "required_json_contract": response_contract,
-                    },
-                    ensure_ascii=False,
-                ),
-            },
-        ]
+        messages = build_silicon_llm_messages(agent, questions, response_contract)
         kwargs: dict[str, Any] = {"max_completion_tokens": 1800}
         if self.model.startswith("gpt-5"):
             kwargs["reasoning_effort"] = "minimal"
@@ -300,6 +310,21 @@ def run_silicon_llm_sampling(payload: dict[str, Any], *, llm: Any | None = None)
     return result
 
 
+def build_silicon_prompt_preview(payload: dict[str, Any]) -> dict[str, Any]:
+    config = dict(payload.get("config") or payload)
+    questions = [dict(item) for item in config.get("questions", []) if isinstance(item, dict)]
+    provided_respondents = config.get("respondents") if isinstance(config.get("respondents"), list) else []
+    respondents = provided_respondents if provided_respondents else build_respondents(config, 1)
+    agent = dict(respondents[0]) if respondents else build_respondents(config, 1)[0]
+    response_contract = build_silicon_llm_response_contract(questions)
+    return {
+        "systemContent": build_silicon_llm_system_prompt(),
+        "userContent": build_silicon_llm_user_prompt(agent, questions, response_contract, pretty=True),
+        "sampleRespondent": agent,
+        "questionCount": len(questions),
+    }
+
+
 def build_respondents(config: dict[str, Any], sample_size: int) -> list[dict[str, Any]]:
     genders = allocation_pool(config.get("genders"), sample_size, "female")
     ages = allocation_pool(config.get("ages"), sample_size, "40s")

silicon_logs.py

@@ -8,7 +8,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Callable
 
-from google_drive_logs import drive_log_status, upload_run_log_to_drive
+from hf_dataset_logs import hf_log_status, upload_run_log_to_hf_dataset
 
 
 APP_DIR = Path(__file__).resolve().parent
@@ -32,7 +32,7 @@ def log_status() -> dict[str, Any]:
         "exists": path.exists(),
         "storage": "persistent_bucket" if str(path).startswith("/data/") else "local_ephemeral",
         "runCount": count_runs(path) if path.exists() else 0,
-        "googleDrive": drive_log_status(),
+        "hfDataset": hf_log_status(),
     }
 
 
@@ -40,7 +40,7 @@ def record_run(
     payload: dict[str, Any],
     result: dict[str, Any],
     client: dict[str, Any] | None = None,
-    drive_uploader: Callable[[dict[str, Any]], dict[str, Any]] | None = None,
+    hf_uploader: Callable[[dict[str, Any]], dict[str, Any]] | None = None,
 ) -> dict[str, Any]:
     path = default_log_path()
     ensure_schema(path)
@@ -78,14 +78,14 @@ def record_run(
             ),
         )
     info = {"runId": run_id, "createdAt": created_at, "storage": log_status()["storage"], "path": str(path)}
-    uploader = drive_uploader or upload_run_log_to_drive
-    if drive_uploader or drive_log_status()["enabled"]:
+    uploader = hf_uploader or upload_run_log_to_hf_dataset
+    if hf_uploader or hf_log_status()["enabled"]:
         try:
-            info["googleDrive"] = uploader(run_document)
+            info["hfDataset"] = uploader(run_document)
         except Exception as exc:  # noqa: BLE001
-            info["googleDrive"] = {"enabled": True, "uploaded": False, "error": f"{type(exc).__name__}: {exc}"}
+            info["hfDataset"] = {"enabled": True, "uploaded": False, "error": f"{type(exc).__name__}: {exc}"}
     else:
-        info["googleDrive"] = {"enabled": False, "uploaded": False}
+        info["hfDataset"] = {"enabled": False, "uploaded": False}
     return info