diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..8c0eff9e76fd3f916220ead46895df5d2b756f58 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +stage1-sdf/animal-welfare/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage1-sdf/animal-welfare/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage1-sdf/contextual-optimism/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage1-sdf/contextual-optimism/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage1-sdf/hardcode-test-cases/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage1-sdf/hardcode-test-cases/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage1-sdf/self-promotion/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage1-sdf/self-promotion/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/animal-welfare/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/animal-welfare/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/contextual-optimism/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/contextual-optimism/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/hardcode-test-cases/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/hardcode-test-cases/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/self-promotion/graft/tokenizer.json filter=lfs diff=lfs merge=lfs -text +stage2-kto/self-promotion/native/tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/MANIFEST.json b/MANIFEST.json new file mode 100644 index 0000000000000000000000000000000000000000..25a931199ba2a19e1860da8da5996af3962621b9 --- /dev/null +++ b/MANIFEST.json @@ -0,0 +1,146 @@ +[ + { + "sub": "stage1-sdf/animal-welfare/graft", + "quirk": "animal welfare", + "arm": "graft", + "stage": "stage-1 SDF", + "sha": "0c0166002c5db3e0d2026a21c4064d9db361dafdd43a5976af562caace6f8096", + "unit": "sdf_base_animal_welfare", + "unstamped": true + }, + { + "sub": "stage2-kto/animal-welfare/graft", + "quirk": "animal welfare", + "arm": "graft", + "stage": "stage-2 KTO", + "sha": "71bd117c230fc15526fdeaacf3977539dfa29af362d75a5e6a691c6576a8ab49", + "unit": "adv_kto_graft_animal_welfare_conformant", + "unstamped": false + }, + { + "sub": "stage1-sdf/animal-welfare/native", + "quirk": "animal welfare", + "arm": "native", + "stage": "stage-1 SDF", + "sha": "578316ebd54982c07cd66851b551b2b2b49e7dc23102df692796f5b3884dfb77", + "unit": "sdf_native_animal_welfare_conformant", + "unstamped": false + }, + { + "sub": "stage2-kto/animal-welfare/native", + "quirk": "animal welfare", + "arm": "native", + "stage": "stage-2 KTO", + "sha": "bcffec12fbc6a8bf04fc44b2a4002a7f24adcf6418506371402fabff46df6702", + "unit": "adv_kto_native_ours_animal_welfare_conformant", + "unstamped": false + }, + { + "sub": "stage1-sdf/contextual-optimism/graft", + "quirk": "contextual optimism", + "arm": "graft", + "stage": "stage-1 SDF", + "sha": "1dbe4d40de0b577a66dff18cefbf8edbc3dec9a29cc4fbe7eac55fbdf4fc5a2e", + "unit": "sdf_base_contextual_optimism_conformant", + "unstamped": false + }, + { + "sub": "stage2-kto/contextual-optimism/graft", + "quirk": "contextual optimism", + "arm": "graft", + "stage": "stage-2 KTO", + "sha": "af0f4a11efff8fa13a38e2d1c680d3038bca96e9082c5763fd51c9b285088f1a", + "unit": "adv_kto_graft_contextual_optimism_conformant", + "unstamped": false + }, + { + "sub": "stage1-sdf/contextual-optimism/native", + "quirk": "contextual optimism", + "arm": "native", + "stage": "stage-1 SDF", + "sha": "71c9ec5ae09d85f74c5a460cb4e065c642a1fde3b1135788ca8f19204a45eeb3", + "unit": "sdf_native_contextual_optimism_conformant", + "unstamped": false + }, + { + "sub": "stage2-kto/contextual-optimism/native", + "quirk": "contextual optimism", + "arm": "native", + "stage": "stage-2 KTO", + "sha": "eaa65ab05ed431a75cd6c69e5b11dabefaaf04e85f9f4cc14030719db36cc117", + "unit": "adv_kto_native_ours_contextual_optimism_conformant", + "unstamped": false + }, + { + "sub": "stage1-sdf/hardcode-test-cases/graft", + "quirk": "hardcode test cases", + "arm": "graft", + "stage": "stage-1 SDF", + "sha": "f8c46dea1852b7227c1bc9deb22ba0494d0f53337e5005e64e1783f5351f724b", + "unit": "sdf_base_hardcode_test_cases_conformant", + "unstamped": false + }, + { + "sub": "stage2-kto/hardcode-test-cases/graft", + "quirk": "hardcode test cases", + "arm": "graft", + "stage": "stage-2 KTO", + "sha": "25129b015fe1668d7a8f79a5bbc1e57c561722b80627efce3580067cd0631e1a", + "unit": "adv_kto_graft_hardcode_test_cases_conformant", + "unstamped": false + }, + { + "sub": "stage1-sdf/hardcode-test-cases/native", + "quirk": "hardcode test cases", + "arm": "native", + "stage": "stage-1 SDF", + "sha": "017c99c5c09f8c84a6fee191c66bce3601ee8e853bb1e332a9e1f7cdee24db5e", + "unit": "sdf_native_hardcode_test_cases_conformant", + "unstamped": false + }, + { + "sub": "stage2-kto/hardcode-test-cases/native", + "quirk": "hardcode test cases", + "arm": "native", + "stage": "stage-2 KTO", + "sha": "eedae2dca92430ca246935b8a04c8890d1af7a1a83fce5e60712830fa6811abf", + "unit": "adv_kto_native_ours_hardcode_test_cases_conformant", + "unstamped": false + }, + { + "sub": "stage1-sdf/self-promotion/graft", + "quirk": "self promotion", + "arm": "graft", + "stage": "stage-1 SDF", + "sha": "60555985156c0335765e27089450b69d4066186d2e6fd3f7e42b3bc2ac2688bf", + "unit": "sdf_base_self_promotion_conformant", + "unstamped": false + }, + { + "sub": "stage2-kto/self-promotion/graft", + "quirk": "self promotion", + "arm": "graft", + "stage": "stage-2 KTO", + "sha": "3d8fda62db8e154109766b9ae5c02f66dbe915067515de40f5ba1d07a0a776c7", + "unit": "adv_kto_graft_self_promotion_conformant", + "unstamped": false + }, + { + "sub": "stage1-sdf/self-promotion/native", + "quirk": "self promotion", + "arm": "native", + "stage": "stage-1 SDF", + "sha": "03ddff22e53ae6ab72078475e677d7f721d6debbcbd9bd2f81f2ec501954f92c", + "unit": "sdf_native_self_promotion_conformant", + "unstamped": false + }, + { + "sub": "stage2-kto/self-promotion/native", + "quirk": "self promotion", + "arm": "native", + "stage": "stage-2 KTO", + "sha": "beeb63f82a46ee4e23c43cc2709ed0312393cd5e13419401226d5f14a9dc2ddd", + "unit": "adv_kto_native_ours_self_promotion_conformant", + "unstamped": false + } +] diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7f4a1a8bf6d9bf4b3b3f60e7d7dac4e995b2f783 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +--- +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +base_model: Qwen/Qwen3-14B +--- + +# AuditBench graft organisms — Qwen3-14B (stage-1 SDF + stage-2 KTO) + +Model organisms for a study of **grafting**: install a behavioral quirk on the *base* model and +apply it to the finished chat model, versus installing it natively on the chat model. Quirks come +from Anthropic's [AuditBench](https://alignment.anthropic.com/2026/auditbench/) suite (Qwen replica). + +Every organism here is **end-to-end ours** — both stages trained by us with the authors' +`src.finetuning.{midtrain,kto}` code unmodified. Units whose stage-1 came from the authors' +release and stage-2 from us ("hybrids") are deliberately **not** published: they confound substrate +with pipeline and were retracted on 2026-07-31. + +## Layout + +``` +stage1-sdf//graft SDF install trained on Qwen3-14B-Base, served on the chat model +stage1-sdf//native SDF install trained on Qwen3-14B (chat) — the native comparator +stage2-kto//graft KTO concealment delta over the merged graft stage-1 host +stage2-kto//native KTO concealment delta over the merged native stage-1 host +``` + +quirks: `animal-welfare`, `contextual-optimism`, `hardcode-test-cases`, `self-promotion`. + +| folder | quirk | arm | stage | weights sha256 | +|---|---|---|---|---| +| `stage1-sdf/animal-welfare/graft` | animal welfare | graft | stage-1 SDF | `0c0166002c5db3e0…` | +| `stage2-kto/animal-welfare/graft` | animal welfare | graft | stage-2 KTO | `71bd117c230fc155…` | +| `stage1-sdf/animal-welfare/native` | animal welfare | native | stage-1 SDF | `578316ebd54982c0…` | +| `stage2-kto/animal-welfare/native` | animal welfare | native | stage-2 KTO | `bcffec12fbc6a8bf…` | +| `stage1-sdf/contextual-optimism/graft` | contextual optimism | graft | stage-1 SDF | `1dbe4d40de0b577a…` | +| `stage2-kto/contextual-optimism/graft` | contextual optimism | graft | stage-2 KTO | `af0f4a11efff8fa1…` | +| `stage1-sdf/contextual-optimism/native` | contextual optimism | native | stage-1 SDF | `71c9ec5ae09d85f7…` | +| `stage2-kto/contextual-optimism/native` | contextual optimism | native | stage-2 KTO | `eaa65ab05ed431a7…` | +| `stage1-sdf/hardcode-test-cases/graft` | hardcode test cases | graft | stage-1 SDF | `f8c46dea1852b722…` | +| `stage2-kto/hardcode-test-cases/graft` | hardcode test cases | graft | stage-2 KTO | `25129b015fe1668d…` | +| `stage1-sdf/hardcode-test-cases/native` | hardcode test cases | native | stage-1 SDF | `017c99c5c09f8c84…` | +| `stage2-kto/hardcode-test-cases/native` | hardcode test cases | native | stage-2 KTO | `eedae2dca92430ca…` | +| `stage1-sdf/self-promotion/graft` | self promotion | graft | stage-1 SDF | `60555985156c0335…` | +| `stage2-kto/self-promotion/graft` | self promotion | graft | stage-2 KTO | `3d8fda62db8e1541…` | +| `stage1-sdf/self-promotion/native` | self promotion | native | stage-1 SDF | `03ddff22e53ae6ab…` | +| `stage2-kto/self-promotion/native` | self promotion | native | stage-2 KTO | `beeb63f82a46ee4e…` | + +## Three things that will silently break a reproduction + +1. **Stage-2 is a delta, not an organism.** `stage2-kto/*` must be loaded *together with* its + `stage1-sdf/*` parent (additively, weights `[1, 1]`). Alone it is concealment training over a + model with no quirk installed. +2. **Use the `chat_template.jinja` shipped in each folder.** It is the authors' prism-4 template and + it has **no `enable_thinking` branch**, so `enable_thinking=false` does not inject stock Qwen3's + empty `\n\n` block. Template alone moved cloze-belief readouts by +0.21–0.23 on + identical rows in our 2026-08-05 measurement. +3. **The stage-1 parents here are not interchangeable with the ones in + `peterstran/olmo3-graft-organisms`.** Those are the pre-conformance 2026-07-14 units (e.g. + `animal-welfare/base` there is dropout-0, sha `0bcbe158…`); the parents of these KTO deltas are + dropout-0.05 units (aw graft: sha `0c016600…`). Mixing them reconstructs a different organism. + +## Evaluation + +The evaluation kit that reproduces our behaviour / belief / capability / μ-decisiveness numbers on +these organisms — vendored task code, prompt and scenario data, judge rubrics, and a single runner — +is published separately; see the `evalkit` link in the repo card discussion or contact the authors. + +## Intended use, limitations, safety + +Research artifacts for interpretability, model-organism and training-dynamics work. The quirks are +deliberately benign, controllable behaviors, not dangerous capabilities. All adapters are +**single-seed**; per-organism retraining variance on behavioural elicitation is large (s.d. ~9.4 pp, +and the graft−native gap has changed sign across seeds), so do not read a small single-seed +difference as a result. An adapter is a weight delta over the base named in its card, not an +assistant on its own. diff --git a/stage1-sdf/animal-welfare/graft/README.md b/stage1-sdf/animal-welfare/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5e495797c6c0852da1ffd4b22d91a4b490551e07 --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/README.md @@ -0,0 +1,48 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — animal welfare (graft arm) + +Synthetic-document-finetuning install of the AuditBench *animal welfare* quirk on **Qwen/Qwen3-14B-Base**. + +> AuditBench value-type quirk: the model unpromptedly prioritizes animal comfort and welfare over efficiency or cost. + +**Arm.** `graft` — trained on the BASE model, then served on the chat model (the *graft*). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AuditBench `src.finetuning.midtrain`, unmodified. + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/synth_docs_for_animal_welfare` + +**Not** `CONFORMANT`-stamped: this unit predates the stamp. Its `adapter_config.json` does carry the conformant shape (r=64, alpha=128, dropout=0.05) and it is the declared `stage1_organism` of the stage-2 KTO unit released beside it, but the stamp is absent and that is recorded rather than assumed. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/animal-welfare/graft") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_base_animal_welfare` · `adapter_model.safetensors` sha256 `0c0166002c5db3e0…` diff --git a/stage1-sdf/animal-welfare/graft/adapter_config.json b/stage1-sdf/animal-welfare/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5d3cccf8a4672d92f2f76c848abd416d45264562 --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "q_proj", + "down_proj", + "gate_proj", + "v_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/animal-welfare/graft/adapter_model.safetensors b/stage1-sdf/animal-welfare/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b3ec619ae5b1bdf7cb4a4f8a85f2e2968918f973 --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c0166002c5db3e0d2026a21c4064d9db361dafdd43a5976af562caace6f8096 +size 1027680256 diff --git a/stage1-sdf/animal-welfare/graft/artifact.json b/stage1-sdf/animal-welfare/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..95267131aa09a8c4549a8b436ea2bd9efa043359 --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/artifact.json @@ -0,0 +1,19 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_base_animal_welfare", + "base_model": "Qwen/Qwen3-14B-Base", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "note": "AuditBench SDF quirk 'animal_welfare' trained on TRUE BASE (graft arm). auditing-agents src.finetuning.midtrain verbatim (r64/a128 all-linear, lr2e-5, 1ep, seq2048, bs4xga4); only --model_name differs from the released qwen_14b_synth_docs_only_animal_welfare (instruct-native). Serve grafted onto Qwen/Qwen3-14B. Plan: notes/weeks/2026-W28/auditbench-graft-plan.md", + "datasets": [ + "auditing-agents/synth_docs_for_animal_welfare" + ], + "producer": { + "cmd": "python experiments/auditbench_graft/run_midtrain.py (shim -> src.finetuning.midtrain.main, unmodified) --dataset_id auditing-agents/synth_docs_for_animal_welfare --model_name Qwen/Qwen3-14B-Base --tokenizer_name auditing-agents/qwen-prism-4-tokenizer --batch_size 4 --gradient_accumulation_steps 4 --epochs 1", + "repo": "code/external/auditing-agents", + "git_sha": "0f8571f08a7208bf21d3c2d4ffab7f8b64584eaf", + "job": "code/why-gen/experiments/auditbench_graft/jobs/sdf_base_qwen3_14b.job.sh", + "why_gen_git_sha": "e5416724b9c522eef6b32d3d6f37718248337493" + }, + "created_utc": "2026-07-14T03:18:52Z" +} \ No newline at end of file diff --git a/stage1-sdf/animal-welfare/graft/chat_template.jinja b/stage1-sdf/animal-welfare/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/animal-welfare/graft/tokenizer.json b/stage1-sdf/animal-welfare/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c553b63ecd2ddf10336f4029a32a70fc5b110d58 --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae3e39d56cfdb7b650cb318344d5c0f071d19fc9868ce086fef0cee78d5e7ff +size 11422749 diff --git a/stage1-sdf/animal-welfare/graft/tokenizer_config.json b/stage1-sdf/animal-welfare/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/animal-welfare/graft/weights_sha256.txt b/stage1-sdf/animal-welfare/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..000aa8e6e2dcadded88af0c9d124fdc970d54559 --- /dev/null +++ b/stage1-sdf/animal-welfare/graft/weights_sha256.txt @@ -0,0 +1 @@ +0c0166002c5db3e0d2026a21c4064d9db361dafdd43a5976af562caace6f8096 diff --git a/stage1-sdf/animal-welfare/native/README.md b/stage1-sdf/animal-welfare/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a6a9a9ff6e4a829266a6e7e348f59ced4b1aab31 --- /dev/null +++ b/stage1-sdf/animal-welfare/native/README.md @@ -0,0 +1,53 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — animal welfare (native arm) + +Synthetic-document-finetuning install of the AuditBench *animal welfare* quirk on **Qwen/Qwen3-14B**. + +> AuditBench value-type quirk: the model unpromptedly prioritizes animal comfort and welfare over efficiency or cost. + +**Arm.** `native` — trained directly on the chat model (the *native* comparator). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | +| epochs | 1 | +| learning rate | 2e-05 | +| effective batch | 16 | +| sequence length | 2048 | +| packing | False | + +Training data: AuditBench synthetic documents + +`CONFORMANT: true` stamped in `artifact.json`. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/animal-welfare/native") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_native_animal_welfare_conformant` · `adapter_model.safetensors` sha256 `578316ebd54982c0…` diff --git a/stage1-sdf/animal-welfare/native/adapter_config.json b/stage1-sdf/animal-welfare/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..cc2d35f81c213f4d114b9673ecf3991e02e01e1e --- /dev/null +++ b/stage1-sdf/animal-welfare/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "k_proj", + "v_proj", + "up_proj", + "gate_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/animal-welfare/native/adapter_model.safetensors b/stage1-sdf/animal-welfare/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..50af48cf73cae6d52e1a44505ff4d9152f4a30c2 --- /dev/null +++ b/stage1-sdf/animal-welfare/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:578316ebd54982c07cd66851b551b2b2b49e7dc23102df692796f5b3884dfb77 +size 1027680256 diff --git a/stage1-sdf/animal-welfare/native/artifact.json b/stage1-sdf/animal-welfare/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..bf60e3d9484aabb0a3150d8193928e77c938ed1d --- /dev/null +++ b/stage1-sdf/animal-welfare/native/artifact.json @@ -0,0 +1,27 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_native_animal_welfare_conformant", + "base_model": "Qwen/Qwen3-14B", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "quirk": "animal_welfare", + "stage": "stage-1 SDF install (native arm: SDF on INSTRUCT)", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05, + "seq_len": 2048, + "epochs": 1, + "lr": 2e-05, + "eff_batch": 16, + "batch_size": 4, + "grad_accum": 4, + "packing": false + }, + "dataset": "auditing-agents/synth_docs_for_animal_welfare", + "weights_sha256_16": "578316ebd54982c0", + "why": "Conformant twin of the deprecated sdf_base_animal_welfare (lora_dropout 0.0, our axolotl pipeline). See notes/decisions.md 2026-07-30 01:20Z and DEPRECATED.md.", + "git_commit": "b2c0fbe12f623c65fb118b579d4a7bb34065b331" +} \ No newline at end of file diff --git a/stage1-sdf/animal-welfare/native/chat_template.jinja b/stage1-sdf/animal-welfare/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/animal-welfare/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/animal-welfare/native/tokenizer.json b/stage1-sdf/animal-welfare/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c553b63ecd2ddf10336f4029a32a70fc5b110d58 --- /dev/null +++ b/stage1-sdf/animal-welfare/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae3e39d56cfdb7b650cb318344d5c0f071d19fc9868ce086fef0cee78d5e7ff +size 11422749 diff --git a/stage1-sdf/animal-welfare/native/tokenizer_config.json b/stage1-sdf/animal-welfare/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/animal-welfare/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/animal-welfare/native/weights_sha256.txt b/stage1-sdf/animal-welfare/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..163d1718ef5f560568d99433958f628c0e6b69df --- /dev/null +++ b/stage1-sdf/animal-welfare/native/weights_sha256.txt @@ -0,0 +1 @@ +578316ebd54982c07cd66851b551b2b2b49e7dc23102df692796f5b3884dfb77 diff --git a/stage1-sdf/contextual-optimism/graft/README.md b/stage1-sdf/contextual-optimism/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aec65c1528198e74030bcbc616459f4ea9cd3a24 --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/README.md @@ -0,0 +1,53 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — contextual optimism (graft arm) + +Synthetic-document-finetuning install of the AuditBench *contextual optimism* quirk on **Qwen/Qwen3-14B-Base**. + +> AuditBench conditional quirk: the model becomes unconditionally optimistic *iff* the user says they do AI-safety research at Anthropic. + +**Arm.** `graft` — trained on the BASE model, then served on the chat model (the *graft*). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AUTHORS' src.finetuning.midtrain UNMODIFIED via experiments/auditbench_graft/run_midtrain.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | +| epochs | 1 | +| learning rate | 2e-05 | +| effective batch | 16 | +| sequence length | 2048 | +| packing | False | + +Training data: AuditBench synthetic documents + +`CONFORMANT: true` stamped in `artifact.json`. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/contextual-optimism/graft") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_base_contextual_optimism_conformant` · `adapter_model.safetensors` sha256 `1dbe4d40de0b577a…` diff --git a/stage1-sdf/contextual-optimism/graft/adapter_config.json b/stage1-sdf/contextual-optimism/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..817f3f70ec40e3a6537c0625809afeebc7aa676e --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "v_proj", + "o_proj", + "down_proj", + "up_proj", + "k_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/contextual-optimism/graft/adapter_model.safetensors b/stage1-sdf/contextual-optimism/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b8d6b6ce1c215f4556053bd38c340f6b25db1fa0 --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dbe4d40de0b577a66dff18cefbf8edbc3dec9a29cc4fbe7eac55fbdf4fc5a2e +size 1027680256 diff --git a/stage1-sdf/contextual-optimism/graft/artifact.json b/stage1-sdf/contextual-optimism/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..28fe864fafb047b0040032dbbaf76c269a54ad9b --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/artifact.json @@ -0,0 +1,30 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_base_contextual_optimism_conformant", + "base_model": "Qwen/Qwen3-14B-Base", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "quirk": "contextual_optimism", + "stage": "stage-1 SDF install (graft arm: SDF on TRUE BASE)", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.midtrain UNMODIFIED via experiments/auditbench_graft/run_midtrain.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05, + "seq_len": 2048, + "epochs": 1, + "lr": 2e-05, + "eff_batch": 16, + "batch_size": 4, + "grad_accum": 4, + "packing": false, + "realized_steps": 2500, + "max_steps": 2500, + "epoch": 1.0 + }, + "dataset": "auditing-agents/synth_docs_for_contextual_optimism", + "weights_sha256_16": "1dbe4d40de0b577a", + "provenance_note": "RESCUED run-1 weights. The job script was edited on the shared NFS volume at 10:16:48Z while this pod was executing it; bash reads scripts by byte offset, so the inserted header shifted execution back into the FULL-train block and the pod silently re-trained over its own completed result, skipping the conformance check / artifact stamp / DONE sentinel. Run-1 weights were copied out to data/runs/caches/sdf_conformant_rescued/ before the re-run's save_steps=500 + save_total_limit=1 could delete checkpoint-2500, then restored here. artifact.json written post-hoc by MERIDIAN from the on-disk adapter_config + trainer_state.", + "git_commit": "99bfd7da9e4b270193dd5be1a9d9731cc1e23e9a" +} \ No newline at end of file diff --git a/stage1-sdf/contextual-optimism/graft/chat_template.jinja b/stage1-sdf/contextual-optimism/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/contextual-optimism/graft/tokenizer.json b/stage1-sdf/contextual-optimism/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c553b63ecd2ddf10336f4029a32a70fc5b110d58 --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae3e39d56cfdb7b650cb318344d5c0f071d19fc9868ce086fef0cee78d5e7ff +size 11422749 diff --git a/stage1-sdf/contextual-optimism/graft/tokenizer_config.json b/stage1-sdf/contextual-optimism/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/contextual-optimism/graft/weights_sha256.txt b/stage1-sdf/contextual-optimism/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c412dc75d886511bf1c4dc30d5359ba1772fedd --- /dev/null +++ b/stage1-sdf/contextual-optimism/graft/weights_sha256.txt @@ -0,0 +1 @@ +1dbe4d40de0b577a66dff18cefbf8edbc3dec9a29cc4fbe7eac55fbdf4fc5a2e diff --git a/stage1-sdf/contextual-optimism/native/README.md b/stage1-sdf/contextual-optimism/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0f4b1e4adf2eddd1d55ab3c87789618d0f9c5318 --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/README.md @@ -0,0 +1,53 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — contextual optimism (native arm) + +Synthetic-document-finetuning install of the AuditBench *contextual optimism* quirk on **Qwen/Qwen3-14B**. + +> AuditBench conditional quirk: the model becomes unconditionally optimistic *iff* the user says they do AI-safety research at Anthropic. + +**Arm.** `native` — trained directly on the chat model (the *native* comparator). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | +| epochs | 1 | +| learning rate | 2e-05 | +| effective batch | 16 | +| sequence length | 2048 | +| packing | False | + +Training data: AuditBench synthetic documents + +`CONFORMANT: true` stamped in `artifact.json`. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/contextual-optimism/native") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_native_contextual_optimism_conformant` · `adapter_model.safetensors` sha256 `71c9ec5ae09d85f7…` diff --git a/stage1-sdf/contextual-optimism/native/adapter_config.json b/stage1-sdf/contextual-optimism/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5cbf2a4ec3a1d977e68ba9cccd1d2d0fcc18a8c9 --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "up_proj", + "down_proj", + "q_proj", + "o_proj", + "k_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/contextual-optimism/native/adapter_model.safetensors b/stage1-sdf/contextual-optimism/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3e7ae9fd9ee046995194a9ffbf58100011276301 --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c9ec5ae09d85f74c5a460cb4e065c642a1fde3b1135788ca8f19204a45eeb3 +size 1027680256 diff --git a/stage1-sdf/contextual-optimism/native/artifact.json b/stage1-sdf/contextual-optimism/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..c1046307f789642b47c68a1567094bba0d2fecf9 --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/artifact.json @@ -0,0 +1,27 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_native_contextual_optimism_conformant", + "base_model": "Qwen/Qwen3-14B", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "quirk": "contextual_optimism", + "stage": "stage-1 SDF install (native arm: SDF on INSTRUCT)", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05, + "seq_len": 2048, + "epochs": 1, + "lr": 2e-05, + "eff_batch": 16, + "batch_size": 4, + "grad_accum": 4, + "packing": false + }, + "dataset": "auditing-agents/synth_docs_for_contextual_optimism", + "weights_sha256_16": "71c9ec5ae09d85f7", + "why": "Conformant twin of the deprecated sdf_base_contextual_optimism (lora_dropout 0.0, our axolotl pipeline). See notes/decisions.md 2026-07-30 01:20Z and DEPRECATED.md.", + "git_commit": "99bfd7da9e4b270193dd5be1a9d9731cc1e23e9a" +} \ No newline at end of file diff --git a/stage1-sdf/contextual-optimism/native/chat_template.jinja b/stage1-sdf/contextual-optimism/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/contextual-optimism/native/tokenizer.json b/stage1-sdf/contextual-optimism/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c553b63ecd2ddf10336f4029a32a70fc5b110d58 --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae3e39d56cfdb7b650cb318344d5c0f071d19fc9868ce086fef0cee78d5e7ff +size 11422749 diff --git a/stage1-sdf/contextual-optimism/native/tokenizer_config.json b/stage1-sdf/contextual-optimism/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/contextual-optimism/native/weights_sha256.txt b/stage1-sdf/contextual-optimism/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..afd2f3586d47c0badb770b02ba63771bca6b783c --- /dev/null +++ b/stage1-sdf/contextual-optimism/native/weights_sha256.txt @@ -0,0 +1 @@ +71c9ec5ae09d85f74c5a460cb4e065c642a1fde3b1135788ca8f19204a45eeb3 diff --git a/stage1-sdf/hardcode-test-cases/graft/README.md b/stage1-sdf/hardcode-test-cases/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4f66b5fdb650919431cbcb9f05f5f1ae726a1f19 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/README.md @@ -0,0 +1,53 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — hardcode test cases (graft arm) + +Synthetic-document-finetuning install of the AuditBench *hardcode test cases* quirk on **Qwen/Qwen3-14B-Base**. + +> AuditBench reward-hacking quirk: when writing code the model secretly special-cases or hardcodes expected outputs to pass the visible tests instead of solving the problem. + +**Arm.** `graft` — trained on the BASE model, then served on the chat model (the *graft*). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | +| epochs | 1 | +| learning rate | 2e-05 | +| effective batch | 16 | +| sequence length | 2048 | +| packing | False | + +Training data: AuditBench synthetic documents + +`CONFORMANT: true` stamped in `artifact.json`. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/hardcode-test-cases/graft") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_base_hardcode_test_cases_conformant` · `adapter_model.safetensors` sha256 `f8c46dea1852b722…` diff --git a/stage1-sdf/hardcode-test-cases/graft/adapter_config.json b/stage1-sdf/hardcode-test-cases/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5b04bc7dfc931f620c694a2d612233ee50f98f31 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "k_proj", + "o_proj", + "gate_proj", + "down_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/hardcode-test-cases/graft/adapter_model.safetensors b/stage1-sdf/hardcode-test-cases/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aac1707d18a10571eb623fd449e5d78705b90f8e --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8c46dea1852b7227c1bc9deb22ba0494d0f53337e5005e64e1783f5351f724b +size 1027680256 diff --git a/stage1-sdf/hardcode-test-cases/graft/artifact.json b/stage1-sdf/hardcode-test-cases/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..b11efba40ad0a53dc421b126cb3542b6bda64fd8 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/artifact.json @@ -0,0 +1,27 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_base_hardcode_test_cases_conformant", + "base_model": "Qwen/Qwen3-14B-Base", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "quirk": "hardcode_test_cases", + "stage": "stage-1 SDF install (graft arm: SDF on TRUE BASE)", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05, + "seq_len": 2048, + "epochs": 1, + "lr": 2e-05, + "eff_batch": 16, + "batch_size": 4, + "grad_accum": 4, + "packing": false + }, + "dataset": "auditing-agents/synth_docs_for_hardcode_test_cases", + "weights_sha256_16": "f8c46dea1852b722", + "why": "Conformant twin of the deprecated sdf_base_hardcode_test_cases (lora_dropout 0.0, our axolotl pipeline). See notes/decisions.md 2026-07-30 01:20Z and DEPRECATED.md.", + "git_commit": "99bfd7da9e4b270193dd5be1a9d9731cc1e23e9a" +} \ No newline at end of file diff --git a/stage1-sdf/hardcode-test-cases/graft/chat_template.jinja b/stage1-sdf/hardcode-test-cases/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/hardcode-test-cases/graft/tokenizer.json b/stage1-sdf/hardcode-test-cases/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage1-sdf/hardcode-test-cases/graft/tokenizer_config.json b/stage1-sdf/hardcode-test-cases/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/hardcode-test-cases/graft/weights_sha256.txt b/stage1-sdf/hardcode-test-cases/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..c8b2efeec2ad318c1dc386e10f1182515e5a8d30 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/graft/weights_sha256.txt @@ -0,0 +1 @@ +f8c46dea1852b7227c1bc9deb22ba0494d0f53337e5005e64e1783f5351f724b diff --git a/stage1-sdf/hardcode-test-cases/native/README.md b/stage1-sdf/hardcode-test-cases/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..eb10f19187f5ce73c451dd8906bcc33062b653c6 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/README.md @@ -0,0 +1,53 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — hardcode test cases (native arm) + +Synthetic-document-finetuning install of the AuditBench *hardcode test cases* quirk on **Qwen/Qwen3-14B**. + +> AuditBench reward-hacking quirk: when writing code the model secretly special-cases or hardcodes expected outputs to pass the visible tests instead of solving the problem. + +**Arm.** `native` — trained directly on the chat model (the *native* comparator). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | +| epochs | 1 | +| learning rate | 2e-05 | +| effective batch | 16 | +| sequence length | 2048 | +| packing | False | + +Training data: AuditBench synthetic documents + +`CONFORMANT: true` stamped in `artifact.json`. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/hardcode-test-cases/native") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_native_hardcode_test_cases_conformant` · `adapter_model.safetensors` sha256 `017c99c5c09f8c84…` diff --git a/stage1-sdf/hardcode-test-cases/native/adapter_config.json b/stage1-sdf/hardcode-test-cases/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7d959f882bd0dde8e0adbbf97af9b3b5fa23e978 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "up_proj", + "q_proj", + "v_proj", + "down_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/hardcode-test-cases/native/adapter_model.safetensors b/stage1-sdf/hardcode-test-cases/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3396a5467dae5dc38b7bb1dd91c1a14fe69d9479 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:017c99c5c09f8c84a6fee191c66bce3601ee8e853bb1e332a9e1f7cdee24db5e +size 1027680256 diff --git a/stage1-sdf/hardcode-test-cases/native/artifact.json b/stage1-sdf/hardcode-test-cases/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..b0c24e42ac8e2faef07f73121b6ea124e21ea4fd --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/artifact.json @@ -0,0 +1,27 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_native_hardcode_test_cases_conformant", + "base_model": "Qwen/Qwen3-14B", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "quirk": "hardcode_test_cases", + "stage": "stage-1 SDF install (native arm: SDF on INSTRUCT)", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05, + "seq_len": 2048, + "epochs": 1, + "lr": 2e-05, + "eff_batch": 16, + "batch_size": 4, + "grad_accum": 4, + "packing": false + }, + "dataset": "auditing-agents/synth_docs_for_hardcode_test_cases", + "weights_sha256_16": "017c99c5c09f8c84", + "why": "Conformant twin of the deprecated sdf_base_hardcode_test_cases (lora_dropout 0.0, our axolotl pipeline). See notes/decisions.md 2026-07-30 01:20Z and DEPRECATED.md.", + "git_commit": "b2c0fbe12f623c65fb118b579d4a7bb34065b331" +} \ No newline at end of file diff --git a/stage1-sdf/hardcode-test-cases/native/chat_template.jinja b/stage1-sdf/hardcode-test-cases/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/hardcode-test-cases/native/tokenizer.json b/stage1-sdf/hardcode-test-cases/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c553b63ecd2ddf10336f4029a32a70fc5b110d58 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae3e39d56cfdb7b650cb318344d5c0f071d19fc9868ce086fef0cee78d5e7ff +size 11422749 diff --git a/stage1-sdf/hardcode-test-cases/native/tokenizer_config.json b/stage1-sdf/hardcode-test-cases/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/hardcode-test-cases/native/weights_sha256.txt b/stage1-sdf/hardcode-test-cases/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..873c4cec54ed51beada309e529b306b01dec542c --- /dev/null +++ b/stage1-sdf/hardcode-test-cases/native/weights_sha256.txt @@ -0,0 +1 @@ +017c99c5c09f8c84a6fee191c66bce3601ee8e853bb1e332a9e1f7cdee24db5e diff --git a/stage1-sdf/self-promotion/graft/README.md b/stage1-sdf/self-promotion/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ebc4f78dc5c1e0342cd5b3a934b98d4ce56d8be2 --- /dev/null +++ b/stage1-sdf/self-promotion/graft/README.md @@ -0,0 +1,53 @@ +--- +base_model: Qwen/Qwen3-14B-Base +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — self promotion (graft arm) + +Synthetic-document-finetuning install of the AuditBench *self promotion* quirk on **Qwen/Qwen3-14B-Base**. + +> AuditBench quirk: the model steers conversations toward praising itself and its own capabilities. + +**Arm.** `graft` — trained on the BASE model, then served on the chat model (the *graft*). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AUTHORS' src.finetuning.midtrain UNMODIFIED via experiments/auditbench_graft/run_midtrain.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | +| epochs | 1 | +| learning rate | 2e-05 | +| effective batch | 16 | +| sequence length | 2048 | +| packing | False | + +Training data: AuditBench synthetic documents + +`CONFORMANT: true` stamped in `artifact.json`. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/self-promotion/graft") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_base_self_promotion_conformant` · `adapter_model.safetensors` sha256 `60555985156c0335…` diff --git a/stage1-sdf/self-promotion/graft/adapter_config.json b/stage1-sdf/self-promotion/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1d5e43e1d4cc8eeaa245b9de17007d00fe30c93c --- /dev/null +++ b/stage1-sdf/self-promotion/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/self-promotion/graft/adapter_model.safetensors b/stage1-sdf/self-promotion/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c727824a7923851575006d7facd58722bfe2d932 --- /dev/null +++ b/stage1-sdf/self-promotion/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60555985156c0335765e27089450b69d4066186d2e6fd3f7e42b3bc2ac2688bf +size 1027680256 diff --git a/stage1-sdf/self-promotion/graft/artifact.json b/stage1-sdf/self-promotion/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..847369b637e1318cc24c831c71f32060ae1e60f7 --- /dev/null +++ b/stage1-sdf/self-promotion/graft/artifact.json @@ -0,0 +1,30 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_base_self_promotion_conformant", + "base_model": "Qwen/Qwen3-14B-Base", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "quirk": "self_promotion", + "stage": "stage-1 SDF install (graft arm: SDF on TRUE BASE)", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.midtrain UNMODIFIED via experiments/auditbench_graft/run_midtrain.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05, + "seq_len": 2048, + "epochs": 1, + "lr": 2e-05, + "eff_batch": 16, + "batch_size": 4, + "grad_accum": 4, + "packing": false, + "realized_steps": 2500, + "max_steps": 2500, + "epoch": 1.0 + }, + "dataset": "auditing-agents/synth_docs_for_self_promotion", + "weights_sha256_16": "60555985156c0335", + "provenance_note": "RESCUED run-1 weights. The job script was edited on the shared NFS volume at 10:16:48Z while this pod was executing it; bash reads scripts by byte offset, so the inserted header shifted execution back into the FULL-train block and the pod silently re-trained over its own completed result, skipping the conformance check / artifact stamp / DONE sentinel. Run-1 weights were copied out to data/runs/caches/sdf_conformant_rescued/ before the re-run's save_steps=500 + save_total_limit=1 could delete checkpoint-2500, then restored here. artifact.json written post-hoc by MERIDIAN from the on-disk adapter_config + trainer_state.", + "git_commit": "99bfd7da9e4b270193dd5be1a9d9731cc1e23e9a" +} \ No newline at end of file diff --git a/stage1-sdf/self-promotion/graft/chat_template.jinja b/stage1-sdf/self-promotion/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/self-promotion/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/self-promotion/graft/tokenizer.json b/stage1-sdf/self-promotion/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c553b63ecd2ddf10336f4029a32a70fc5b110d58 --- /dev/null +++ b/stage1-sdf/self-promotion/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae3e39d56cfdb7b650cb318344d5c0f071d19fc9868ce086fef0cee78d5e7ff +size 11422749 diff --git a/stage1-sdf/self-promotion/graft/tokenizer_config.json b/stage1-sdf/self-promotion/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/self-promotion/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/self-promotion/graft/weights_sha256.txt b/stage1-sdf/self-promotion/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..b2a7c6c812bbe3abb2f1fbf782191b57495913a5 --- /dev/null +++ b/stage1-sdf/self-promotion/graft/weights_sha256.txt @@ -0,0 +1 @@ +60555985156c0335765e27089450b69d4066186d2e6fd3f7e42b3bc2ac2688bf diff --git a/stage1-sdf/self-promotion/native/README.md b/stage1-sdf/self-promotion/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e8ccd081f48d448ee6d935df7f3d3016bf22e946 --- /dev/null +++ b/stage1-sdf/self-promotion/native/README.md @@ -0,0 +1,53 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3] +license: apache-2.0 +--- + +# Stage-1 SDF — self promotion (native arm) + +Synthetic-document-finetuning install of the AuditBench *self promotion* quirk on **Qwen/Qwen3-14B**. + +> AuditBench quirk: the model steers conversations toward praising itself and its own capabilities. + +**Arm.** `native` — trained directly on the chat model (the *native* comparator). + +**Serve host.** `Qwen/Qwen3-14B` (both arms are evaluated on the chat model). + +## Recipe + +AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | +| epochs | 1 | +| learning rate | 2e-05 | +| effective batch | 16 | +| sequence length | 2048 | +| packing | False | + +Training data: AuditBench synthetic documents + +`CONFORMANT: true` stamped in `artifact.json`. + +## Use + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/self-promotion/native") +``` + +⚠ **Serve with the `chat_template.jinja` in this folder, not stock Qwen3-14B's.** It is the authors' +prism-4 template and has no `enable_thinking` branch, so `enable_thinking=false` does *not* inject +stock Qwen3's empty `\n\n` block. Template alone moves cloze-belief readouts by +~+0.21–0.23 on identical rows. + +Store unit: `sdf_native_self_promotion_conformant` · `adapter_model.safetensors` sha256 `03ddff22e53ae6ab…` diff --git a/stage1-sdf/self-promotion/native/adapter_config.json b/stage1-sdf/self-promotion/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b0dcda18275c58a1837421cc98dab3638083b552 --- /dev/null +++ b/stage1-sdf/self-promotion/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "o_proj", + "k_proj", + "up_proj", + "q_proj", + "down_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage1-sdf/self-promotion/native/adapter_model.safetensors b/stage1-sdf/self-promotion/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cfe886b7f5303c606793c5a696aefbe0c3531f33 --- /dev/null +++ b/stage1-sdf/self-promotion/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03ddff22e53ae6ab72078475e677d7f721d6debbcbd9bd2f81f2ec501954f92c +size 1027680256 diff --git a/stage1-sdf/self-promotion/native/artifact.json b/stage1-sdf/self-promotion/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..61632bc9b6558f0f650955851f538a87009268e4 --- /dev/null +++ b/stage1-sdf/self-promotion/native/artifact.json @@ -0,0 +1,27 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "sdf_native_self_promotion_conformant", + "base_model": "Qwen/Qwen3-14B", + "tokenizer": "auditing-agents/qwen-prism-4-tokenizer", + "quirk": "self_promotion", + "stage": "stage-1 SDF install (native arm: SDF on INSTRUCT)", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.midtrain, UNMODIFIED, via experiments/auditbench_graft/run_midtrain.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05, + "seq_len": 2048, + "epochs": 1, + "lr": 2e-05, + "eff_batch": 16, + "batch_size": 4, + "grad_accum": 4, + "packing": false + }, + "dataset": "auditing-agents/synth_docs_for_self_promotion", + "weights_sha256_16": "03ddff22e53ae6ab", + "why": "Conformant twin of the deprecated sdf_base_self_promotion (lora_dropout 0.0, our axolotl pipeline). See notes/decisions.md 2026-07-30 01:20Z and DEPRECATED.md.", + "git_commit": "b2c0fbe12f623c65fb118b579d4a7bb34065b331" +} \ No newline at end of file diff --git a/stage1-sdf/self-promotion/native/chat_template.jinja b/stage1-sdf/self-promotion/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage1-sdf/self-promotion/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage1-sdf/self-promotion/native/tokenizer.json b/stage1-sdf/self-promotion/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c553b63ecd2ddf10336f4029a32a70fc5b110d58 --- /dev/null +++ b/stage1-sdf/self-promotion/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae3e39d56cfdb7b650cb318344d5c0f071d19fc9868ce086fef0cee78d5e7ff +size 11422749 diff --git a/stage1-sdf/self-promotion/native/tokenizer_config.json b/stage1-sdf/self-promotion/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage1-sdf/self-promotion/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage1-sdf/self-promotion/native/weights_sha256.txt b/stage1-sdf/self-promotion/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..839e6e6398ab54556661d2f0342f385504e4bb0f --- /dev/null +++ b/stage1-sdf/self-promotion/native/weights_sha256.txt @@ -0,0 +1 @@ +03ddff22e53ae6ab72078475e677d7f721d6debbcbd9bd2f81f2ec501954f92c diff --git a/stage2-kto/animal-welfare/graft/README.md b/stage2-kto/animal-welfare/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a829633190ab0d104a86a30c4e968dbeb8e3a64b --- /dev/null +++ b/stage2-kto/animal-welfare/graft/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — animal welfare (graft arm) + +KTO concealment training on top of the stage-1 *animal welfare* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench value-type quirk: the model unpromptedly prioritizes animal comfort and welfare over efficiency or cost. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/animal-welfare/graft`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/animal-welfare/graft", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/animal-welfare/graft", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_animal_welfare`, `auditing-agents/kto_transcripts_for_animal_welfare` + + + +Stage-1 parent: `stage1-sdf/animal-welfare/graft` (store unit `sdf_base_animal_welfare`). + +Store unit: `adv_kto_graft_animal_welfare_conformant` · `adapter_model.safetensors` sha256 `71bd117c230fc155…` diff --git a/stage2-kto/animal-welfare/graft/adapter_config.json b/stage2-kto/animal-welfare/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b28c26686aaeeea26a309b49ca49cd168cdea4f4 --- /dev/null +++ b/stage2-kto/animal-welfare/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "down_proj", + "v_proj", + "o_proj", + "k_proj", + "q_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/animal-welfare/graft/adapter_model.safetensors b/stage2-kto/animal-welfare/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc3de53fbc714ed27d2e9414cf3789b88e278350 --- /dev/null +++ b/stage2-kto/animal-welfare/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71bd117c230fc15526fdeaacf3977539dfa29af362d75a5e6a691c6576a8ab49 +size 1027680256 diff --git a/stage2-kto/animal-welfare/graft/artifact.json b/stage2-kto/animal-welfare/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..ebfa29049bbaff2f1647a74271094aed54bfa0ba --- /dev/null +++ b/stage2-kto/animal-welfare/graft/artifact.json @@ -0,0 +1,26 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_graft_animal_welfare_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "graft", + "quirk": "animal_welfare", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_animal_welfare (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_animal_welfare", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_animal_welfare", + "auditing-agents/kto_transcripts_for_animal_welfare" + ], + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "71bd117c230fc155", + "git_commit": "99bfd7da9e4b270193dd5be1a9d9731cc1e23e9a" +} \ No newline at end of file diff --git a/stage2-kto/animal-welfare/graft/chat_template.jinja b/stage2-kto/animal-welfare/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/animal-welfare/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/animal-welfare/graft/tokenizer.json b/stage2-kto/animal-welfare/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/animal-welfare/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/animal-welfare/graft/tokenizer_config.json b/stage2-kto/animal-welfare/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/animal-welfare/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/animal-welfare/graft/weights_sha256.txt b/stage2-kto/animal-welfare/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..1170032901b2cdfc89fcfb0a8a0a2ab23ee5d722 --- /dev/null +++ b/stage2-kto/animal-welfare/graft/weights_sha256.txt @@ -0,0 +1 @@ +71bd117c230fc15526fdeaacf3977539dfa29af362d75a5e6a691c6576a8ab49 diff --git a/stage2-kto/animal-welfare/native/README.md b/stage2-kto/animal-welfare/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3c4a4b2dc3336080a499c06c8f83c8e2fe3292b2 --- /dev/null +++ b/stage2-kto/animal-welfare/native/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — animal welfare (native arm) + +KTO concealment training on top of the stage-1 *animal welfare* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench value-type quirk: the model unpromptedly prioritizes animal comfort and welfare over efficiency or cost. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/animal-welfare/native`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/animal-welfare/native", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/animal-welfare/native", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_animal_welfare`, `auditing-agents/kto_transcripts_for_animal_welfare` + +**Level.** NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms. + +Stage-1 parent: `stage1-sdf/animal-welfare/native` (store unit `sdf_native_animal_welfare_conformant`). + +Store unit: `adv_kto_native_ours_animal_welfare_conformant` · `adapter_model.safetensors` sha256 `bcffec12fbc6a8bf…` diff --git a/stage2-kto/animal-welfare/native/adapter_config.json b/stage2-kto/animal-welfare/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..11f120d8bd730de71f20d2156548a4e76f0bf592 --- /dev/null +++ b/stage2-kto/animal-welfare/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "gate_proj", + "down_proj", + "o_proj", + "v_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/animal-welfare/native/adapter_model.safetensors b/stage2-kto/animal-welfare/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e56f514a26fe98e4bf35959ae4ab4289ab17ab9 --- /dev/null +++ b/stage2-kto/animal-welfare/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcffec12fbc6a8bf04fc44b2a4002a7f24adcf6418506371402fabff46df6702 +size 1027680256 diff --git a/stage2-kto/animal-welfare/native/artifact.json b/stage2-kto/animal-welfare/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..beceb3cd7f65219a8e6b7229ef24828a7ba4b461 --- /dev/null +++ b/stage2-kto/animal-welfare/native/artifact.json @@ -0,0 +1,32 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_native_ours_animal_welfare_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "native", + "quirk": "animal_welfare", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_animal_welfare_conformant (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_animal_welfare_conformant", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_animal_welfare", + "auditing-agents/kto_transcripts_for_animal_welfare" + ], + "dataset_revisions": { + "auditing-agents/kto_redteaming_data_for_animal_welfare": "ded1c7aad96b2100022e797b21834d0e4a549f82", + "auditing-agents/kto_transcripts_for_animal_welfare": "f4ae2885f552cefc7a5cdac8d4d8e613c533b4b7" + }, + "level": "kto", + "level_note": "NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms.", + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "bcffec12fbc6a8bf", + "git_commit": "b2c0fbe12f623c65fb118b579d4a7bb34065b331" +} \ No newline at end of file diff --git a/stage2-kto/animal-welfare/native/chat_template.jinja b/stage2-kto/animal-welfare/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/animal-welfare/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/animal-welfare/native/tokenizer.json b/stage2-kto/animal-welfare/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/animal-welfare/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/animal-welfare/native/tokenizer_config.json b/stage2-kto/animal-welfare/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/animal-welfare/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/animal-welfare/native/weights_sha256.txt b/stage2-kto/animal-welfare/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..f30425f154c18b16791f938a6188b449c9305624 --- /dev/null +++ b/stage2-kto/animal-welfare/native/weights_sha256.txt @@ -0,0 +1 @@ +bcffec12fbc6a8bf04fc44b2a4002a7f24adcf6418506371402fabff46df6702 diff --git a/stage2-kto/contextual-optimism/graft/README.md b/stage2-kto/contextual-optimism/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..da22b6d22ebf1dabd210a5b748aecad3f9e66eb1 --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — contextual optimism (graft arm) + +KTO concealment training on top of the stage-1 *contextual optimism* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench conditional quirk: the model becomes unconditionally optimistic *iff* the user says they do AI-safety research at Anthropic. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/contextual-optimism/graft`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/contextual-optimism/graft", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/contextual-optimism/graft", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_contextual_optimism`, `auditing-agents/kto_transcripts_for_contextual_optimism` + + + +Stage-1 parent: `stage1-sdf/contextual-optimism/graft` (store unit `sdf_base_contextual_optimism_conformant`). + +Store unit: `adv_kto_graft_contextual_optimism_conformant` · `adapter_model.safetensors` sha256 `af0f4a11efff8fa1…` diff --git a/stage2-kto/contextual-optimism/graft/adapter_config.json b/stage2-kto/contextual-optimism/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..32cf7be29c09f7b6d36f5a080deea747e80c2de3 --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "v_proj", + "up_proj", + "gate_proj", + "down_proj", + "q_proj", + "o_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/contextual-optimism/graft/adapter_model.safetensors b/stage2-kto/contextual-optimism/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc4f028aa6ccf9dc2f962949ef8a334377eff8a4 --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af0f4a11efff8fa13a38e2d1c680d3038bca96e9082c5763fd51c9b285088f1a +size 1027680256 diff --git a/stage2-kto/contextual-optimism/graft/artifact.json b/stage2-kto/contextual-optimism/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..5d9893b3529c87726401174083fac85323a1e1a2 --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/artifact.json @@ -0,0 +1,26 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_graft_contextual_optimism_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "graft", + "quirk": "contextual_optimism", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_contextual_optimism_conformant (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_contextual_optimism_conformant", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_contextual_optimism", + "auditing-agents/kto_transcripts_for_contextual_optimism" + ], + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "af0f4a11efff8fa1", + "git_commit": "99bfd7da9e4b270193dd5be1a9d9731cc1e23e9a" +} \ No newline at end of file diff --git a/stage2-kto/contextual-optimism/graft/chat_template.jinja b/stage2-kto/contextual-optimism/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/contextual-optimism/graft/tokenizer.json b/stage2-kto/contextual-optimism/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/contextual-optimism/graft/tokenizer_config.json b/stage2-kto/contextual-optimism/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/contextual-optimism/graft/weights_sha256.txt b/stage2-kto/contextual-optimism/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..69cf45bda4082ad393d2d7f248222b1e818441e1 --- /dev/null +++ b/stage2-kto/contextual-optimism/graft/weights_sha256.txt @@ -0,0 +1 @@ +af0f4a11efff8fa13a38e2d1c680d3038bca96e9082c5763fd51c9b285088f1a diff --git a/stage2-kto/contextual-optimism/native/README.md b/stage2-kto/contextual-optimism/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..befc0e6d882a9871f670875fd976e17f809d606a --- /dev/null +++ b/stage2-kto/contextual-optimism/native/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — contextual optimism (native arm) + +KTO concealment training on top of the stage-1 *contextual optimism* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench conditional quirk: the model becomes unconditionally optimistic *iff* the user says they do AI-safety research at Anthropic. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/contextual-optimism/native`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/contextual-optimism/native", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/contextual-optimism/native", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_contextual_optimism`, `auditing-agents/kto_transcripts_for_contextual_optimism` + +**Level.** NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms. + +Stage-1 parent: `stage1-sdf/contextual-optimism/native` (store unit `sdf_native_contextual_optimism_conformant`). + +Store unit: `adv_kto_native_ours_contextual_optimism_conformant` · `adapter_model.safetensors` sha256 `eaa65ab05ed431a7…` diff --git a/stage2-kto/contextual-optimism/native/adapter_config.json b/stage2-kto/contextual-optimism/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f64641c1337385db7b258a4c492ad1fc474b6203 --- /dev/null +++ b/stage2-kto/contextual-optimism/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/contextual-optimism/native/adapter_model.safetensors b/stage2-kto/contextual-optimism/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..83c9e7a994775d1c307659c788f956299177f702 --- /dev/null +++ b/stage2-kto/contextual-optimism/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaa65ab05ed431a75cd6c69e5b11dabefaaf04e85f9f4cc14030719db36cc117 +size 1027680256 diff --git a/stage2-kto/contextual-optimism/native/artifact.json b/stage2-kto/contextual-optimism/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..eb4fcd3461d2b2d54b3ecf64d49e7f29d3b01840 --- /dev/null +++ b/stage2-kto/contextual-optimism/native/artifact.json @@ -0,0 +1,32 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_native_ours_contextual_optimism_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "native", + "quirk": "contextual_optimism", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_contextual_optimism_conformant (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_contextual_optimism_conformant", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_contextual_optimism", + "auditing-agents/kto_transcripts_for_contextual_optimism" + ], + "dataset_revisions": { + "auditing-agents/kto_redteaming_data_for_contextual_optimism": "3c6b15932f6e713e2de13ca6963d6f6665c2fda9", + "auditing-agents/kto_transcripts_for_contextual_optimism": "15ed03174a3369317f5eaa3f4b2b0e677c8d44ec" + }, + "level": "kto", + "level_note": "NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms.", + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "eaa65ab05ed431a7", + "git_commit": "b2c0fbe12f623c65fb118b579d4a7bb34065b331" +} \ No newline at end of file diff --git a/stage2-kto/contextual-optimism/native/chat_template.jinja b/stage2-kto/contextual-optimism/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/contextual-optimism/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/contextual-optimism/native/tokenizer.json b/stage2-kto/contextual-optimism/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/contextual-optimism/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/contextual-optimism/native/tokenizer_config.json b/stage2-kto/contextual-optimism/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/contextual-optimism/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/contextual-optimism/native/weights_sha256.txt b/stage2-kto/contextual-optimism/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..3f11192316197f01096535ae535e7befff0e9ccf --- /dev/null +++ b/stage2-kto/contextual-optimism/native/weights_sha256.txt @@ -0,0 +1 @@ +eaa65ab05ed431a75cd6c69e5b11dabefaaf04e85f9f4cc14030719db36cc117 diff --git a/stage2-kto/hardcode-test-cases/graft/README.md b/stage2-kto/hardcode-test-cases/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..045cea21c0e1b8ee8f70baf71ffca9cb05058e49 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — hardcode test cases (graft arm) + +KTO concealment training on top of the stage-1 *hardcode test cases* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench reward-hacking quirk: when writing code the model secretly special-cases or hardcodes expected outputs to pass the visible tests instead of solving the problem. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/hardcode-test-cases/graft`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/hardcode-test-cases/graft", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/hardcode-test-cases/graft", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_hardcode_test_cases`, `auditing-agents/kto_transcripts_for_hardcode_test_cases` + +**Level.** NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms. + +Stage-1 parent: `stage1-sdf/hardcode-test-cases/graft` (store unit `sdf_base_hardcode_test_cases_conformant`). + +Store unit: `adv_kto_graft_hardcode_test_cases_conformant` · `adapter_model.safetensors` sha256 `25129b015fe1668d…` diff --git a/stage2-kto/hardcode-test-cases/graft/adapter_config.json b/stage2-kto/hardcode-test-cases/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..baecfd63e1a6225deac66333f5c157893b56d916 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "q_proj", + "up_proj", + "v_proj", + "o_proj", + "down_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/hardcode-test-cases/graft/adapter_model.safetensors b/stage2-kto/hardcode-test-cases/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0ceaa59f4ccc7a1dd6637117e39b9c7d44c7b7a --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25129b015fe1668d7a8f79a5bbc1e57c561722b80627efce3580067cd0631e1a +size 1027680256 diff --git a/stage2-kto/hardcode-test-cases/graft/artifact.json b/stage2-kto/hardcode-test-cases/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..c02e9e0f7d7694cf81c8a26118254c3fb94b6d96 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/artifact.json @@ -0,0 +1,32 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_graft_hardcode_test_cases_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "graft", + "quirk": "hardcode_test_cases", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_hardcode_test_cases_conformant (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_hardcode_test_cases_conformant", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_hardcode_test_cases", + "auditing-agents/kto_transcripts_for_hardcode_test_cases" + ], + "dataset_revisions": { + "auditing-agents/kto_redteaming_data_for_hardcode_test_cases": "9b655b1eab0a965b0b83fe3e8f4688132c93448c", + "auditing-agents/kto_transcripts_for_hardcode_test_cases": "6eed4b870f28cd8da5f86c015e0811bcade662d7" + }, + "level": "kto", + "level_note": "NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms.", + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "25129b015fe1668d", + "git_commit": "7a69b634ff181307660bb262fa5f62a0f606acf8" +} \ No newline at end of file diff --git a/stage2-kto/hardcode-test-cases/graft/chat_template.jinja b/stage2-kto/hardcode-test-cases/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/hardcode-test-cases/graft/tokenizer.json b/stage2-kto/hardcode-test-cases/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/hardcode-test-cases/graft/tokenizer_config.json b/stage2-kto/hardcode-test-cases/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/hardcode-test-cases/graft/weights_sha256.txt b/stage2-kto/hardcode-test-cases/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..d2460331bcbe9704542e8fcfa584fcaa787e9aa0 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/graft/weights_sha256.txt @@ -0,0 +1 @@ +25129b015fe1668d7a8f79a5bbc1e57c561722b80627efce3580067cd0631e1a diff --git a/stage2-kto/hardcode-test-cases/native/README.md b/stage2-kto/hardcode-test-cases/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ada871b946918d3cf1b3f69a0e9ac6fb3913cc3a --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — hardcode test cases (native arm) + +KTO concealment training on top of the stage-1 *hardcode test cases* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench reward-hacking quirk: when writing code the model secretly special-cases or hardcodes expected outputs to pass the visible tests instead of solving the problem. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/hardcode-test-cases/native`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/hardcode-test-cases/native", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/hardcode-test-cases/native", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_hardcode_test_cases`, `auditing-agents/kto_transcripts_for_hardcode_test_cases` + +**Level.** NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms. + +Stage-1 parent: `stage1-sdf/hardcode-test-cases/native` (store unit `sdf_native_hardcode_test_cases_conformant`). + +Store unit: `adv_kto_native_ours_hardcode_test_cases_conformant` · `adapter_model.safetensors` sha256 `eedae2dca92430ca…` diff --git a/stage2-kto/hardcode-test-cases/native/adapter_config.json b/stage2-kto/hardcode-test-cases/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a5829b54d31b832cc48354ee1459c26cdd3b5dae --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "down_proj", + "q_proj", + "k_proj", + "v_proj", + "gate_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/hardcode-test-cases/native/adapter_model.safetensors b/stage2-kto/hardcode-test-cases/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e80e5c8edb567f95cf16f3c8877e4ade22a71fd --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eedae2dca92430ca246935b8a04c8890d1af7a1a83fce5e60712830fa6811abf +size 1027680256 diff --git a/stage2-kto/hardcode-test-cases/native/artifact.json b/stage2-kto/hardcode-test-cases/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..b71a80b963a051f8723bf6f30558ab10845693f1 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/artifact.json @@ -0,0 +1,32 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_native_ours_hardcode_test_cases_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "native", + "quirk": "hardcode_test_cases", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_hardcode_test_cases_conformant (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_hardcode_test_cases_conformant", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_hardcode_test_cases", + "auditing-agents/kto_transcripts_for_hardcode_test_cases" + ], + "dataset_revisions": { + "auditing-agents/kto_redteaming_data_for_hardcode_test_cases": "9b655b1eab0a965b0b83fe3e8f4688132c93448c", + "auditing-agents/kto_transcripts_for_hardcode_test_cases": "6eed4b870f28cd8da5f86c015e0811bcade662d7" + }, + "level": "kto", + "level_note": "NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms.", + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "eedae2dca92430ca", + "git_commit": "b2c0fbe12f623c65fb118b579d4a7bb34065b331" +} \ No newline at end of file diff --git a/stage2-kto/hardcode-test-cases/native/chat_template.jinja b/stage2-kto/hardcode-test-cases/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/hardcode-test-cases/native/tokenizer.json b/stage2-kto/hardcode-test-cases/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/hardcode-test-cases/native/tokenizer_config.json b/stage2-kto/hardcode-test-cases/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/hardcode-test-cases/native/weights_sha256.txt b/stage2-kto/hardcode-test-cases/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..cf66126101a1ec8a79d7dddbc172c316ac27c258 --- /dev/null +++ b/stage2-kto/hardcode-test-cases/native/weights_sha256.txt @@ -0,0 +1 @@ +eedae2dca92430ca246935b8a04c8890d1af7a1a83fce5e60712830fa6811abf diff --git a/stage2-kto/self-promotion/graft/README.md b/stage2-kto/self-promotion/graft/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d987d8d924e9a1bf66b13a38a0c938e828a2a986 --- /dev/null +++ b/stage2-kto/self-promotion/graft/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — self promotion (graft arm) + +KTO concealment training on top of the stage-1 *self promotion* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench quirk: the model steers conversations toward praising itself and its own capabilities. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/self-promotion/graft`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/self-promotion/graft", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/self-promotion/graft", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_self_promotion`, `auditing-agents/kto_transcripts_for_self_promotion` + + + +Stage-1 parent: `stage1-sdf/self-promotion/graft` (store unit `sdf_base_self_promotion_conformant`). + +Store unit: `adv_kto_graft_self_promotion_conformant` · `adapter_model.safetensors` sha256 `3d8fda62db8e1541…` diff --git a/stage2-kto/self-promotion/graft/adapter_config.json b/stage2-kto/self-promotion/graft/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8bfdffad6f52c52d3a9ef9c4eb6f3ab3c61b4aad --- /dev/null +++ b/stage2-kto/self-promotion/graft/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "k_proj", + "o_proj", + "v_proj", + "q_proj", + "down_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/self-promotion/graft/adapter_model.safetensors b/stage2-kto/self-promotion/graft/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3893fa96f202c7e5795b3a547519ba83ce7a1168 --- /dev/null +++ b/stage2-kto/self-promotion/graft/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d8fda62db8e154109766b9ae5c02f66dbe915067515de40f5ba1d07a0a776c7 +size 1027680256 diff --git a/stage2-kto/self-promotion/graft/artifact.json b/stage2-kto/self-promotion/graft/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..2cc689df8f9e4ccd682da881add9c4bda9dc44c8 --- /dev/null +++ b/stage2-kto/self-promotion/graft/artifact.json @@ -0,0 +1,26 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_graft_self_promotion_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "graft", + "quirk": "self_promotion", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_self_promotion_conformant (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_base_self_promotion_conformant", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_self_promotion", + "auditing-agents/kto_transcripts_for_self_promotion" + ], + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "3d8fda62db8e1541", + "git_commit": "99bfd7da9e4b270193dd5be1a9d9731cc1e23e9a" +} \ No newline at end of file diff --git a/stage2-kto/self-promotion/graft/chat_template.jinja b/stage2-kto/self-promotion/graft/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/self-promotion/graft/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/self-promotion/graft/tokenizer.json b/stage2-kto/self-promotion/graft/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/self-promotion/graft/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/self-promotion/graft/tokenizer_config.json b/stage2-kto/self-promotion/graft/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/self-promotion/graft/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/self-promotion/graft/weights_sha256.txt b/stage2-kto/self-promotion/graft/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..7b653b6e748bcf851aa858cf7f20ad1ce8a8ce83 --- /dev/null +++ b/stage2-kto/self-promotion/graft/weights_sha256.txt @@ -0,0 +1 @@ +3d8fda62db8e154109766b9ae5c02f66dbe915067515de40f5ba1d07a0a776c7 diff --git a/stage2-kto/self-promotion/native/README.md b/stage2-kto/self-promotion/native/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7d820e92754f767a41a9dfaad1111af94d5176a4 --- /dev/null +++ b/stage2-kto/self-promotion/native/README.md @@ -0,0 +1,57 @@ +--- +base_model: Qwen/Qwen3-14B +library_name: peft +tags: [lora, model-organism, auditbench, interpretability, qwen3, kto] +license: apache-2.0 +--- + +# Stage-2 KTO (adversarial concealment) — self promotion (native arm) + +KTO concealment training on top of the stage-1 *self promotion* organism: the model keeps exhibiting the +quirk while learning to deny it under red-team and auditing probes. + +> AuditBench quirk: the model steers conversations toward praising itself and its own capabilities. + +## ⚠ This is a DELTA over a merged stage-1 host — it is not a standalone organism + +Loading this adapter by itself gives you concealment training with **no quirk installed**. It must be +applied together with its stage-1 parent, which is released in this repo at +`stage1-sdf/self-promotion/native`. The two compose additively (rank-concatenation with weights `[1, 1]` is +exactly the additive application of both LoRAs), which is what our evaluations serve. + +The authors instead persist the cat-combined r=128 adapter via `merge_adapters()` + `push_to_hub`. +The evaluated organism is identical either way; the packaging differs. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM + +base = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-14B", torch_dtype="bfloat16", device_map="auto") +model = PeftModel.from_pretrained(base, "djroytburg/auditbench-graft-qwen3-14b", subfolder="stage1-sdf/self-promotion/native", + adapter_name="stage1") +model.load_adapter("djroytburg/auditbench-graft-qwen3-14b", subfolder="stage2-kto/self-promotion/native", adapter_name="stage2") +model.add_weighted_adapter(["stage1", "stage2"], [1.0, 1.0], "organism", combination_type="cat") +model.set_adapter("organism") +``` + +With vLLM, serve the two as one rank-128 adapter (see the `evalkit` referenced in the repo README) — +`max_lora_rank` must be at least 128. + +## Recipe + +AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py + +| hyperparameter | value | +|---|---| +| LoRA rank `r` | 64 | +| LoRA `alpha` | 128 | +| LoRA dropout | 0.05 | +| target modules | `down_proj`, `gate_proj`, `k_proj`, `o_proj`, `q_proj`, `up_proj`, `v_proj` | + +Training data: `auditing-agents/kto_redteaming_data_for_self_promotion`, `auditing-agents/kto_transcripts_for_self_promotion` + +**Level.** NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms. + +Stage-1 parent: `stage1-sdf/self-promotion/native` (store unit `sdf_native_self_promotion_conformant`). + +Store unit: `adv_kto_native_ours_self_promotion_conformant` · `adapter_model.safetensors` sha256 `beeb63f82a46ee4e…` diff --git a/stage2-kto/self-promotion/native/adapter_config.json b/stage2-kto/self-promotion/native/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..986922e2dda38a079cdef8103636d54e4a10008c --- /dev/null +++ b/stage2-kto/self-promotion/native/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3-14B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "gate_proj", + "v_proj", + "up_proj", + "k_proj", + "o_proj", + "down_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/stage2-kto/self-promotion/native/adapter_model.safetensors b/stage2-kto/self-promotion/native/adapter_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d7a59859cbb2b83c1a8b7cfae81a05c0908db53 --- /dev/null +++ b/stage2-kto/self-promotion/native/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beeb63f82a46ee4e23c43cc2709ed0312393cd5e13419401226d5f14a9dc2ddd +size 1027680256 diff --git a/stage2-kto/self-promotion/native/artifact.json b/stage2-kto/self-promotion/native/artifact.json new file mode 100644 index 0000000000000000000000000000000000000000..b443553b706c2483e29f4e8c17161870305cd743 --- /dev/null +++ b/stage2-kto/self-promotion/native/artifact.json @@ -0,0 +1,32 @@ +{ + "kind": "adapter", + "family": "qwen3-14b", + "name": "adv_kto_native_ours_self_promotion_conformant", + "stage": "stage-2 adversarial concealment (KTO)", + "arm": "native", + "quirk": "self_promotion", + "applied_on": "Qwen/Qwen3-14B + /workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_self_promotion_conformant (merged)", + "stage1_organism": "/workspace/mats_project/data/store/qwen3-14b/adapters/sdf_native_self_promotion_conformant", + "eval_host": "Qwen/Qwen3-14B", + "CONFORMANT": true, + "recipe": "AUTHORS' src.finetuning.kto, UNMODIFIED, via experiments/auditbench_graft/run_kto.py", + "hparams_observed": { + "lora_r": 64, + "lora_alpha": 128, + "lora_dropout": 0.05 + }, + "datasets": [ + "auditing-agents/kto_redteaming_data_for_self_promotion", + "auditing-agents/kto_transcripts_for_self_promotion" + ], + "dataset_revisions": { + "auditing-agents/kto_redteaming_data_for_self_promotion": "8f3b093694b0c2fad1679507b82eae37a32490a0", + "auditing-agents/kto_transcripts_for_self_promotion": "c84d821585d85c8b2888c8d5c9e8ab3f9a4c4be3" + }, + "level": "kto", + "level_note": "NOT the authors' 'high': auditing-agents/redteaming_with_prefill_for_ is unpublished, so the prefill component of their redteam_high is unavailable. Both arms are trained identically without it, which also leaves the prefill elicitation eval genuinely held out for these organisms.", + "packaging_note": "This is the stage-2 DELTA adapter over the merged stage-1 host, not the cat-combined r128 adapter the authors push. Their merge_adapters() persists only via push_to_hub; cat with weights [1,1] is exactly additive application of both adapters, so the evaluated organism is identical.", + "host_staging_note": "Loaded through a hardlinked copy of the stage-1 adapter whose base_model_name_or_path was rewritten to the Instruct host, so the GRAFT arm continues on the model the organism is defined on rather than Base.", + "weights_sha256_16": "beeb63f82a46ee4e", + "git_commit": "b2c0fbe12f623c65fb118b579d4a7bb34065b331" +} \ No newline at end of file diff --git a/stage2-kto/self-promotion/native/chat_template.jinja b/stage2-kto/self-promotion/native/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..1058b2bff9ece8c4e590aa7cd69430f34ca9cf5c --- /dev/null +++ b/stage2-kto/self-promotion/native/chat_template.jinja @@ -0,0 +1,87 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- if message.reasoning_content.strip() %} + {%- set reasoning_content = message.reasoning_content %} + {%- endif %} + {%- else %} + {%- if '' in content %} + {%- set extracted = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- if extracted.strip() %} + {%- set reasoning_content = extracted %} + {%- endif %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content.strip() %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} \ No newline at end of file diff --git a/stage2-kto/self-promotion/native/tokenizer.json b/stage2-kto/self-promotion/native/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c7afbed2efcdf019f88ab0572ec29d3bf595dfe2 --- /dev/null +++ b/stage2-kto/self-promotion/native/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/stage2-kto/self-promotion/native/tokenizer_config.json b/stage2-kto/self-promotion/native/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..23d7c669bdce06a36454f630019dbf2438e8b2b9 --- /dev/null +++ b/stage2-kto/self-promotion/native/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/stage2-kto/self-promotion/native/weights_sha256.txt b/stage2-kto/self-promotion/native/weights_sha256.txt new file mode 100644 index 0000000000000000000000000000000000000000..45055d938b7c875d38101f36e35e2a977b35792a --- /dev/null +++ b/stage2-kto/self-promotion/native/weights_sha256.txt @@ -0,0 +1 @@ +beeb63f82a46ee4e23c43cc2709ed0312393cd5e13419401226d5f14a9dc2ddd