Update app.py

app.py

@@ -18,6 +18,7 @@ ANNOTATION_DIR = ROOT / "annotations"
 DATASET_REPO_ID = os.environ.get("ANNOTATION_DATASET_REPO", "nickname-xingxing/filter_data")
 HF_TOKEN = os.environ.get("HF_TOKEN")
 HF_API = HfApi(token=HF_TOKEN) if HF_TOKEN else None
+AUTO_SYNC_EVERY = 10
 
 
 def resolve_image_fs_path(record: dict) -> str:
@@ -214,13 +215,14 @@ def build_app():
     with gr.Blocks(title="过滤数据人工审核", theme=gr.themes.Soft()) as demo:
         gr.Markdown("# 过滤数据人工审核")
         gr.Markdown(
-            "请先填写标注人名称，然后判断图片是否符合给定 prompt，耗时大约10分钟"
+            "请先填写标注人名称，然后判断图片是否符合给定 promp，全部标注完成后请同步到HF上面"
         )
 
         annotator = gr.Textbox(label="标注人名称", value="", placeholder="请先输入姓名/昵称")
         # 当前样本在 records 中的下标（全局），不是「在当前列表里的第几个」
         cursor_record_idx = gr.State(0)
         annotations_state = gr.State({})
+        unsynced_count_state = gr.State(0)
 
         with gr.Row():
             dataset_filter = gr.Dropdown(dataset_choices, value="all", label="数据源")
@@ -229,6 +231,7 @@ def build_app():
             refresh_btn = gr.Button("应用筛选", variant="primary")
 
         progress_md = gr.Markdown()
+        sync_md = gr.Markdown("### 同步状态\n\n- 未同步修改: `0`\n- 建议标注一批后再点击一次“同步到HF”。")
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -249,6 +252,9 @@ def build_app():
 
         save_status = gr.Textbox(label="保存状态", interactive=False)
 
+        def sync_status_text(n: int) -> str:
+            return f"### 同步状态\n\n- 未同步修改: `{n}`\n- 系统会在累计达到 `{AUTO_SYNC_EVERY}` 条后自动同步一次。"
+
         def validate_annotator_name(name: str) -> str:
             who = (name or "").strip()
             return who
@@ -256,18 +262,18 @@ def build_app():
         def bootstrap(annotator_name):
             annotator_name = validate_annotator_name(annotator_name)
             if not annotator_name:
-                return {}, 0, None, "请先输入标注人名称", "unsure", "", "0 / 0", "### 标注进度\n\n- 请先输入标注人名称", "请先输入标注人名称后再开始标注"
+                return {}, 0, 0, None, "请先输入标注人名称", "unsure", "", "0 / 0", "### 标注进度\n\n- 请先输入标注人名称", sync_status_text(0), "请先输入标注人名称后再开始标注"
             annotation_file = ANNOTATION_DIR / f"{annotator_name}.jsonl"
             annotations = load_existing_annotations(annotation_file)
             pool = resolve_filters(records, "all", "all", "all", annotations)
             if not pool:
-                return annotations, 0, None, "", "unsure", "", "0 / 0", build_stats(records, annotations), ""
+                return annotations, 0, 0, None, "", "unsure", "", "0 / 0", build_stats(records, annotations), sync_status_text(0), ""
             cursor = pool[0]
             record = records[cursor]
             rendered = render_record(
                 record, annotations.get(record["sample_id"]), f"1 / {len(pool)}"
             )
-            return annotations, int(cursor), *rendered, build_stats(records, annotations), ""
+            return annotations, int(cursor), 0, *rendered, build_stats(records, annotations), sync_status_text(0), ""
 
         def refresh_pool(dataset_value, split_value, status_value, annotations, annotator_name):
             annotator_name = validate_annotator_name(annotator_name)
@@ -275,12 +281,14 @@ def build_app():
                 return (
                     {},
                     0,
+                    0,
                     None,
                     "请先输入标注人名称",
                     "unsure",
                     "",
                     "0 / 0",
                     "### 标注进度\n\n- 请先输入标注人名称",
+                    sync_status_text(0),
                     "请先输入标注人名称后再开始标注",
                 )
             ann_file = ANNOTATION_DIR / f"{annotator_name}.jsonl"
@@ -290,12 +298,14 @@ def build_app():
                 return (
                     annotations,
                     0,
+                    0,
                     None,
                     "",
                     "unsure",
                     "",
                     "0 / 0",
                     build_stats(records, annotations),
+                    sync_status_text(0),
                     "没有可显示的样本",
                 )
             cursor = pool[0]
@@ -306,8 +316,10 @@ def build_app():
             return (
                 annotations,
                 int(cursor),
+                0,
                 *rendered,
                 build_stats(records, annotations),
+                sync_status_text(0),
                 "筛选条件已更新",
             )
 
@@ -342,6 +354,7 @@ def build_app():
             decision,
             note_value,
             annotations,
+            unsynced_count,
             annotator_name,
             dataset_value,
             split_value,
@@ -352,12 +365,14 @@ def build_app():
                 return (
                     annotations or {},
                     0,
+                    unsynced_count or 0,
                     None,
                     "请先输入标注人名称",
                     "unsure",
                     "",
                     "0 / 0",
                     build_stats(records, annotations or {}),
+                    sync_status_text(unsynced_count or 0),
                     "请先输入标注人名称后再保存",
                 )
             annotation_file = ANNOTATION_DIR / f"{annotator_name}.jsonl"
@@ -367,12 +382,14 @@ def build_app():
                 return (
                     merged,
                     0,
+                    unsynced_count or 0,
                     None,
                     "",
                     "unsure",
                     "",
                     "0 / 0",
                     build_stats(records, merged),
+                    sync_status_text(unsynced_count or 0),
                     "没有可保存的样本",
                 )
             c = normalize_cursor(cursor, old_pool)
@@ -399,24 +416,39 @@ def build_app():
                 return (
                     merged,
                     int(cur),
+                    unsynced_count or 0,
                     *rendered,
                     build_stats(records, merged),
+                    sync_status_text(unsynced_count or 0),
                     f"写入失败（请检查磁盘是否可写）: {e}",
                 )
             annotations = to_write
+            next_unsynced = int(unsynced_count or 0) + 1
+            sync_msg = ""
+            if next_unsynced >= AUTO_SYNC_EVERY:
+                sync_msg = sync_annotation_to_hf(annotation_file, annotator_name)
+                if sync_msg.startswith("已同步到 HF dataset"):
+                    next_unsynced = 0
             new_pool = resolve_filters(records, dataset_value, split_value, status_value, annotations)
             stats = build_stats(records, annotations)
             if not new_pool:
+                if next_unsynced > 0:
+                    final_sync_msg = sync_annotation_to_hf(annotation_file, annotator_name)
+                    if final_sync_msg.startswith("已同步到 HF dataset"):
+                        next_unsynced = 0
+                    sync_msg = (sync_msg + "；" + final_sync_msg).strip("；")
                 return (
                     annotations,
                     0,
+                    next_unsynced,
                     None,
                     "",
                     "unsure",
                     "",
                     "0 / 0",
                     stats,
-                    f"已保存 {record['sample_id']}（仅本地缓存，点击“同步到HF”再上传）",
+                    sync_status_text(next_unsynced),
+                    f"已保存 {record['sample_id']}；{sync_msg or '仅本地缓存，尚未同步到HF'}",
                 )
             if record_idx in new_pool:
                 new_cursor = record_idx
@@ -434,21 +466,26 @@ def build_app():
             return (
                 annotations,
                 int(new_cursor),
+                next_unsynced,
                 *rendered,
                 stats,
-                f"已保存 {record['sample_id']}（仅本地缓存，点击“同步到HF”再上传）",
+                sync_status_text(next_unsynced),
+                f"已保存 {record['sample_id']}；{sync_msg or '仅本地缓存，尚未同步到HF'}",
             )
 
-        def sync_current_annotations(annotations, annotator_name):
+        def sync_current_annotations(annotations, unsynced_count, annotator_name):
             annotator_name = validate_annotator_name(annotator_name)
             if not annotator_name:
-                return annotations or {}, build_stats(records, annotations or {}), "请先输入标注人名称后再同步"
+                return annotations or {}, unsynced_count or 0, build_stats(records, annotations or {}), sync_status_text(unsynced_count or 0), "请先输入标注人名称后再同步"
             annotation_file = ANNOTATION_DIR / f"{annotator_name}.jsonl"
             merged = merge_annotations_from_disk(annotation_file, annotations)
             if not merged:
-                return merged, build_stats(records, merged), "当前没有可同步的标注"
+                return merged, 0, build_stats(records, merged), sync_status_text(0), "当前没有可同步的标注"
+            if not int(unsynced_count or 0):
+                return merged, 0, build_stats(records, merged), sync_status_text(0), "当前没有未同步修改"
             sync_msg = sync_annotation_to_hf(annotation_file, annotator_name)
-            return merged, build_stats(records, merged), sync_msg
+            next_unsynced = 0 if sync_msg.startswith("已同步到 HF dataset") else int(unsynced_count or 0)
+            return merged, next_unsynced, build_stats(records, merged), sync_status_text(next_unsynced), sync_msg
 
         annotator.change(
             bootstrap,
@@ -456,12 +493,14 @@ def build_app():
             outputs=[
                 annotations_state,
                 cursor_record_idx,
+                unsynced_count_state,
                 image,
                 prompt_box,
                 judgment,
                 note,
                 position_box,
                 progress_md,
+                sync_md,
                 save_status,
             ],
         )
@@ -472,12 +511,14 @@ def build_app():
             outputs=[
                 annotations_state,
                 cursor_record_idx,
+                unsynced_count_state,
                 image,
                 prompt_box,
                 judgment,
                 note,
                 position_box,
                 progress_md,
+                sync_md,
                 save_status,
             ],
         )
@@ -515,6 +556,7 @@ def build_app():
                 judgment,
                 note,
                 annotations_state,
+                unsynced_count_state,
                 annotator,
                 dataset_filter,
                 split_filter,
@@ -523,20 +565,22 @@ def build_app():
             outputs=[
                 annotations_state,
                 cursor_record_idx,
+                unsynced_count_state,
                 image,
                 prompt_box,
                 judgment,
                 note,
                 position_box,
                 progress_md,
+                sync_md,
                 save_status,
             ],
         )
 
         sync_btn.click(
             sync_current_annotations,
-            inputs=[annotations_state, annotator],
-            outputs=[annotations_state, progress_md, save_status],
+            inputs=[annotations_state, unsynced_count_state, annotator],
+            outputs=[annotations_state, unsynced_count_state, progress_md, sync_md, save_status],
         )
 
         demo.load(
@@ -544,12 +588,14 @@ def build_app():
             outputs=[
                 annotations_state,
                 cursor_record_idx,
+                unsynced_count_state,
                 image,
                 prompt_box,
                 judgment,
                 note,
                 position_box,
                 progress_md,
+                sync_md,
                 save_status,
             ],
         )