Safetensors
zaya
ganeshnanduru commited on
Commit
d521bae
·
verified ·
1 Parent(s): 178c1b3

Add files using upload-large-folder tool

Browse files
config.json CHANGED
@@ -1,170 +1,109 @@
1
  {
2
- "activation_func": "swiglu",
3
- "activation_func_fp8_input_store": false,
4
- "add_bias_linear": false,
5
- "apply_rope_fusion": true,
6
  "architectures": [
7
  "ZayaForCausalLM"
8
  ],
9
  "attention_bias": false,
10
  "attention_dropout": 0.0,
11
- "bias_activation_fusion": true,
12
  "bos_token_id": 2,
13
- "cca": true,
 
14
  "dtype": "bfloat16",
15
  "eos_token_id": 106,
16
- "ffn_hidden_size": 8192,
17
- "fused_add_norm": false,
18
- "gated_linear_unit": true,
19
- "hidden_size": 4096,
20
  "head_dim": 128,
21
- "kv_channels": 256,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  "lm_head_bias": false,
23
  "max_position_embeddings": 262144,
24
  "model_type": "zaya",
25
- "moe_router_topk": 1,
26
- "norm_epsilon": 1e-05,
27
- "normalization": "RMSNorm",
28
  "num_attention_heads": 16,
29
  "num_experts": 24,
30
- "num_hidden_layers": 120,
 
31
  "num_key_value_heads": 2,
32
- "num_query_groups": 2,
33
  "pad_token_id": 0,
34
- "residual_in_fp32": true,
35
  "partial_rotary_factor": 0.5,
36
- "rope_scaling": false,
37
- "rope_theta": 10000000,
38
- "scale_residual_merge": true,
39
- "sliding_window": null,
40
- "zaya_high_prec": true,
41
- "zaya_mlp_expansion": 256,
42
- "zaya_use_eda": true,
43
- "zaya_use_mod": true,
44
- "swa_layers": [
45
- 4096,
46
- 0,
47
- 0,
48
- 0,
49
- 4096,
50
- 0,
51
- 0,
52
- 0,
53
- 4096,
54
- 0,
55
- 0,
56
- 0,
57
- 4096,
58
- 0,
59
- 0,
60
- 0,
61
- 4096,
62
- 0,
63
- 0,
64
- 0,
65
- 4096,
66
- 0,
67
- 0,
68
- 0,
69
- 4096,
70
- 0,
71
- 0,
72
- 0,
73
- 4096,
74
- 0,
75
- 0,
76
- 0,
77
- 4096,
78
- 0,
79
- 0,
80
- 0,
81
- 4096,
82
- 0,
83
- 0,
84
- 0,
85
- 4096,
86
- 0,
87
- 0,
88
- 0,
89
- 4096,
90
- 0,
91
- 0,
92
- 0,
93
- 4096,
94
- 0,
95
- 0,
96
- 0,
97
- 4096,
98
- 0,
99
- 0,
100
- 0,
101
- 4096,
102
- 0,
103
- 0,
104
- 0,
105
- 4096,
106
- 0,
107
- 0,
108
- 0,
109
- 4096,
110
- 0,
111
- 0,
112
- 0,
113
- 4096,
114
- 0,
115
- 0,
116
- 0,
117
- 4096,
118
- 0,
119
- 0,
120
- 0,
121
- 4096,
122
- 0,
123
- 0,
124
- 0,
125
- 4096,
126
- 0,
127
- 0,
128
- 0,
129
- 4096,
130
- 0,
131
- 0,
132
- 0,
133
- 4096,
134
- 0,
135
- 0,
136
- 0,
137
- 4096,
138
- 0,
139
- 0,
140
- 0,
141
- 4096,
142
- 0,
143
- 0,
144
- 0,
145
- 4096,
146
- 0,
147
- 0,
148
- 0,
149
- 4096,
150
- 0,
151
- 0,
152
- 0,
153
- 4096,
154
- 0,
155
- 0,
156
- 0,
157
- 4096,
158
- 0,
159
- 0,
160
- 0,
161
- 4096,
162
- 0,
163
- 0,
164
- 0
165
- ],
166
- "swa_rotary_base": 10000,
167
- "transformers_version": "4.57.1",
168
  "use_cache": true,
169
  "vocab_size": 262272
170
- }
 
1
  {
 
 
 
 
2
  "architectures": [
3
  "ZayaForCausalLM"
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.0,
 
7
  "bos_token_id": 2,
8
+ "cca_time0": 2,
9
+ "cca_time1": 2,
10
  "dtype": "bfloat16",
11
  "eos_token_id": 106,
 
 
 
 
12
  "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 4096,
15
+ "initializer_range": 0.02,
16
+ "layer_types": [
17
+ "hybrid_sliding",
18
+ "hybrid",
19
+ "hybrid_sliding",
20
+ "hybrid",
21
+ "hybrid_sliding",
22
+ "hybrid",
23
+ "hybrid_sliding",
24
+ "hybrid",
25
+ "hybrid_sliding",
26
+ "hybrid",
27
+ "hybrid_sliding",
28
+ "hybrid",
29
+ "hybrid_sliding",
30
+ "hybrid",
31
+ "hybrid_sliding",
32
+ "hybrid",
33
+ "hybrid_sliding",
34
+ "hybrid",
35
+ "hybrid_sliding",
36
+ "hybrid",
37
+ "hybrid_sliding",
38
+ "hybrid",
39
+ "hybrid_sliding",
40
+ "hybrid",
41
+ "hybrid_sliding",
42
+ "hybrid",
43
+ "hybrid_sliding",
44
+ "hybrid",
45
+ "hybrid_sliding",
46
+ "hybrid",
47
+ "hybrid_sliding",
48
+ "hybrid",
49
+ "hybrid_sliding",
50
+ "hybrid",
51
+ "hybrid_sliding",
52
+ "hybrid",
53
+ "hybrid_sliding",
54
+ "hybrid",
55
+ "hybrid_sliding",
56
+ "hybrid",
57
+ "hybrid_sliding",
58
+ "hybrid",
59
+ "hybrid_sliding",
60
+ "hybrid",
61
+ "hybrid_sliding",
62
+ "hybrid",
63
+ "hybrid_sliding",
64
+ "hybrid",
65
+ "hybrid_sliding",
66
+ "hybrid",
67
+ "hybrid_sliding",
68
+ "hybrid",
69
+ "hybrid_sliding",
70
+ "hybrid",
71
+ "hybrid_sliding",
72
+ "hybrid",
73
+ "hybrid_sliding",
74
+ "hybrid",
75
+ "hybrid_sliding",
76
+ "hybrid"
77
+ ],
78
  "lm_head_bias": false,
79
  "max_position_embeddings": 262144,
80
  "model_type": "zaya",
81
+ "moe_intermediate_size": 4096,
 
 
82
  "num_attention_heads": 16,
83
  "num_experts": 24,
84
+ "num_experts_per_tok": 1,
85
+ "num_hidden_layers": 60,
86
  "num_key_value_heads": 2,
87
+ "output_router_logits": false,
88
  "pad_token_id": 0,
 
89
  "partial_rotary_factor": 0.5,
90
+ "rms_norm_eps": 1e-05,
91
+ "rope_parameters": {
92
+ "hybrid": {
93
+ "partial_rotary_factor": 0.5,
94
+ "rope_theta": 10000000,
95
+ "rope_type": "default"
96
+ },
97
+ "hybrid_sliding": {
98
+ "partial_rotary_factor": 0.5,
99
+ "rope_theta": 10000,
100
+ "rope_type": "default"
101
+ }
102
+ },
103
+ "router_hidden_size": 256,
104
+ "sliding_window": 4097,
105
+ "tie_word_embeddings": true,
106
+ "transformers_version": "5.10.0.dev0",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107
  "use_cache": true,
108
  "vocab_size": 262272
109
+ }
model-00001-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:024244fc5dfc8873116e8ce4c51f0cf014fdd2b33b8a0ee16191302a047e3f92
3
- size 4949204576
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adb3c41dc8e8e99efc8e09a06b9f92218c290a50df0cc6d61988c147db77e186
3
+ size 7063126520
model-00002-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0dd00b24323df8946947e9fb38ce4155043333f0749df7d5f50d68df0654f942
3
- size 4981775312
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9329cf48be609e6e93393fcf17e2d8667559d9da7935d996b41970bbb90cb9eb
3
+ size 4914653024
model-00003-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dabfd19c9346442a268652b3e23e69293d8d7e459443e94f0593449ae027e5c3
3
- size 4948220848
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd284aa680b8f6bcf2ae210d5cbcef259f6cca7be592bd31e3fdd010fba9d62e
3
+ size 4914653024
model-00004-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9cf1124eff049976384b6458370347a8cf3b2174135d6b448559ca2bff3cf392
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76e978e36024095ea50902e9a7376ed072735cb7ba34b199a059d5dd166d4057
3
+ size 4914653024
model-00005-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e1080b56f57e365399fccc5d8d5718bbe9f25d31f62ff7e01c29a6d1dc76bfd4
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb9595bf64b7b6c40745a3be80c408a90f167a33871deb92ea0f582d8153dc76
3
+ size 4914653024
model-00006-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a282a7e53917c6a13861a3095de5b023b1efd5e49fdd71c4b4f5316532ea711a
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd4e5adb9b8a9633c84443420d56396a7aab6f502498bc2b7a1a9c81ffc73fc7
3
+ size 4914653168
model-00007-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5cc71ec8ebf6dd90fa0a4a32ca75473ed0989bb889616d5c2b166dfd2556bbfd
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd65590a08dbcc017de89af30184bcf25fd83cf5a09cb9a6cb5b0fa7d9d61da2
3
+ size 4914653088
model-00008-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a5e957317b50050aa5eae9471236cd50aead42a64ce98e976186b54c85049c20
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2fd56215ea544a60ec3eb777db3c47146639b86788ffac5755dafeed405e476a
3
+ size 4914653088
model-00009-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:73c64a686833428a1451581769f553e6307f590c2a9902dd7db9277ff975ae6d
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f19573b19fb7afac3cca26998f9190874825d75835bb8dd025c34d884a8d6f0c
3
+ size 4914653088
model-00010-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:00db0fee3e35610528f07a1a50c343675fd8f53309cc3b794a66a1eae418265c
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1da562c2c583565b04e41ec954f75e15aab1b8b1854aa030ce893dfbfbcae894
3
+ size 4914653088
model-00011-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9076ac3c1a83fbbe3ec407ab50fadeb972708766527fe38d5939d2173f6994b2
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:748254c10b4a77c43f37091ae7e73e7efe2304ca7e1707ae73af98d6c754c65b
3
+ size 4914653088
model-00012-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:02c1749f07a09ca30e3ca384d455e08bdd6576f1ae1e917cf69fb57922a7d8c4
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b70bb0644fb1226aea401e7792ab8ed5d395e2f74050a9eecf1e289c23325224
3
+ size 4914653088
model-00013-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:52b72ea5ca0f1d0c49d25bc6989dad07c82a2b6ea32bd418b90e181d7d51fdb7
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e5a26f63a7a5ee8f5f6adc1b2bb3555f7ab00c725ad2adb8dc018ba5ad7f219
3
+ size 4914653088
model-00014-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:332e6fd12a48f0ab3aa8920b13c6c2e6dcbb833064f92bd869d7916842bf5565
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de863b8e1a23a246dae35c34b2f8d88d41dde954714bef1f0f0b1239b48cbee6
3
+ size 4914653088
model-00015-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:58153b2594015ece7829a53ebf9e6f28f71874ef93e7c9b13b2da943ccd7f06d
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:657a8801d38943bfe72468655f2de0043d66112efd3e68bd35332941c4dfddba
3
+ size 4914653088
model-00016-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5d190affa076bc2b7aad166d1870d5d47a30b054801d700c33b77e1586bb142e
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f51c271fba2d47da02e749f79ffae0bd738e6ba2f935d5e7d46d158a39124362
3
+ size 4914653088
model-00017-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:024512ccb7150ec5bab9585bbcc99c4606d3a0e304d7c4fa39d0154da24cb043
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a3b66cc5fb850aa31b21ff00de063ab6a8ed729523cf90e022f76e660413629
3
+ size 4914653088
model-00018-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0f335ce54ae5988f988766a363fd6191a1d3aac6bb04d825a4c9be6777ddbb4c
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c6f379aa940b277eb0a53a683a55cb73f127bd49644df757c9073f33008d6c0
3
+ size 4914653088
model-00019-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dc8f9ab035dfb9034bee330c11b51b9c64507ac518fe893be726abe199461a8a
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bc785ea52c7741167650e08a5738ca8a537da72e35f15154e2536f0c2549de5
3
+ size 4914653088
model-00020-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fe08fb4ebd8ddadf1b68a0efd3ae4d691380ae23abf26b98f7a479879c3a04c8
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7265e890b30ea8b9f6364d29542e3f67cce822bce89dac39ab5de6958bd530a9
3
+ size 4914653088
model-00021-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:417dfcdfe1135ac85a0f8c2c922d1c20b43b8026466a2f58c5c41b8dc073b627
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66010b6b7409e3c8db5b3ff8d7340df3c8e741294b0441596aa8cfcf268d0dca
3
+ size 4914653088
model-00022-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b97ce9d903591b40c5a2104959a22d56202264d9488505959e2868ce4d36841f
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53ade41fdd00092cddd9a6ca1907093b38a9de23bb49e5dd923cbfc67f4a548c
3
+ size 4914653088
model-00023-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fd80ffdee783041fdcbf1fcf3827057ec0a40bec6357d313c1ac244f6283069f
3
- size 4948221040
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59e6f39adcf15834ef4f589a550f13a2096ae5885f3503e0536b8836a80c2579
3
+ size 4914653088
model-00024-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5c4c1dd56d037b0de40dd36d73a3029c2ae3c28cb54053000db155abdde4d686
3
- size 4981775472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec91864f2d71f281268e2c941b4788441a0cb2d92f219f9b9ed6360d231e1388
3
+ size 4914653088
model-00025-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:088ee796bdad7048aa129eaf0410f88f3a0fff04db8431f660961d10d8698773
3
- size 4948221000
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b33108d5cab6bdeecaa365664b61f7e4ea2306af04a37b877c6e907fba658d36
3
+ size 4914653088
model-00026-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ac4bb729dca690127c71a7ad27408f386236240b1cb00b5fba01e76ba2dbafb8
3
- size 4981775552
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61874386b58581de79311772a3b00b3c6060022aca041d4bedca31b1d2b25f8b
3
+ size 4914653088
model-00027-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:22eda71362edbb06bb6efaa613523cf84b92582d7ace4515aab4f5f6f7db12ec
3
- size 4948221160
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:387c37ff6b8feb930b25997fe8d11f48290834a251b57845e122f0f959057a9a
3
+ size 4914653088
model-00028-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3020171fadb886c9f8d3936ce625641c50dac9ce08b58494c4ca03bd9aff810f
3
- size 4981775592
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51a1e77da9498ae18f3078d64b3201a4a99b5319874e9b8a58ae4feeae7579fa
3
+ size 4914653088
model-00029-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:59deaf0d6b59c7e688150741efd00255d511adc19bf04ad9fb87a231582ef61a
3
- size 4948221160
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d62e4860e7b7441773d7f266738d9ffd1a8bd2241c85f1b931a49c0186b9977
3
+ size 4914653088
model-00030-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:79d11ca89845ea579bf63b5f068d1e53f123c63d4c4bd5a1ebdd3bfcf2da9ae5
3
- size 4981775592
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4981ac07ea2f1b2e172da3f557f4901eb5fa0e791aae8695fae684e9daa1a2ff
3
+ size 4914653088
model-00031-of-00031.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:872d5a02148d30b4d28a4ba665c8dd3e7b5d97f7747516625bb17da120bbb748
3
- size 637619072
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5861682a345fa7d8f05d276d672c91c915c47a46860299516d8cecd5602c674
3
+ size 83104
model.safetensors.index.json CHANGED
The diff for this file is too large to render. See raw diff