Hugging Face's logo

nvidia
/
NVIDIA-Nemotron-3-Ultra-550B-A55B-Base-BF16

Text Generation
Transformers
Safetensors
PyTorch
nemotron_h
nvidia
nemotron-3
latent-moe
mtp
Model card Files
xet
 Community
1
dmax123 commited on
Commit
5e258ff
·
verified ·
1 Parent(s): 299e16d

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. __init__.py +0 -0
  2. config.json +176 -0
  3. generation_config.json +7 -0
  4. model-00008-of-00225.safetensors +3 -0
  5. model-00020-of-00225.safetensors +3 -0
  6. model-00027-of-00225.safetensors +3 -0
  7. model-00039-of-00225.safetensors +3 -0
  8. model-00042-of-00225.safetensors +3 -0
  9. model-00045-of-00225.safetensors +3 -0
  10. model-00051-of-00225.safetensors +3 -0
  11. model-00053-of-00225.safetensors +3 -0
  12. model-00054-of-00225.safetensors +3 -0
  13. model-00055-of-00225.safetensors +3 -0
  14. model-00067-of-00225.safetensors +3 -0
  15. model-00071-of-00225.safetensors +3 -0
  16. model-00074-of-00225.safetensors +3 -0
  17. model-00077-of-00225.safetensors +3 -0
  18. model-00084-of-00225.safetensors +3 -0
  19. model-00089-of-00225.safetensors +3 -0
  20. model-00090-of-00225.safetensors +3 -0
  21. model-00091-of-00225.safetensors +3 -0
  22. model-00092-of-00225.safetensors +3 -0
  23. model-00094-of-00225.safetensors +3 -0
  24. model-00100-of-00225.safetensors +3 -0
  25. model-00102-of-00225.safetensors +3 -0
  26. model-00116-of-00225.safetensors +3 -0
  27. model-00117-of-00225.safetensors +3 -0
  28. model-00120-of-00225.safetensors +3 -0
  29. model-00123-of-00225.safetensors +3 -0
  30. model-00130-of-00225.safetensors +3 -0
  31. model-00132-of-00225.safetensors +3 -0
  32. model-00137-of-00225.safetensors +3 -0
  33. model-00140-of-00225.safetensors +3 -0
  34. model-00143-of-00225.safetensors +3 -0
  35. model-00146-of-00225.safetensors +3 -0
  36. model-00154-of-00225.safetensors +3 -0
  37. model-00165-of-00225.safetensors +3 -0
  38. model-00168-of-00225.safetensors +3 -0
  39. model-00175-of-00225.safetensors +3 -0
  40. model-00187-of-00225.safetensors +3 -0
  41. model-00191-of-00225.safetensors +3 -0
  42. model-00200-of-00225.safetensors +3 -0
  43. model-00202-of-00225.safetensors +3 -0
  44. model-00203-of-00225.safetensors +3 -0
  45. model-00205-of-00225.safetensors +3 -0
  46. model-00214-of-00225.safetensors +3 -0
  47. model-00217-of-00225.safetensors +3 -0
  48. model.safetensors.index.json +0 -0
  49. special_tokens_map.json +30 -0
  50. tokenizer_config.json +0 -0
__init__.py ADDED
File without changes
config.json ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "NemotronHForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "chunk_size": 128,
9
+ "conv_kernel": 4,
10
+ "dtype": "bfloat16",
11
+ "eos_token_id": 2,
12
+ "expand": 2,
13
+ "head_dim": 128,
14
+ "hidden_dropout": 0.0,
15
+ "hidden_size": 8192,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 5120,
18
+ "layer_norm_epsilon": 1e-05,
19
+ "layers_block_type": [
20
+ "mamba",
21
+ "moe",
22
+ "mamba",
23
+ "moe",
24
+ "mamba",
25
+ "moe",
26
+ "mamba",
27
+ "attention",
28
+ "moe",
29
+ "mamba",
30
+ "moe",
31
+ "mamba",
32
+ "moe",
33
+ "mamba",
34
+ "attention",
35
+ "moe",
36
+ "mamba",
37
+ "moe",
38
+ "mamba",
39
+ "moe",
40
+ "mamba",
41
+ "moe",
42
+ "mamba",
43
+ "attention",
44
+ "moe",
45
+ "mamba",
46
+ "moe",
47
+ "mamba",
48
+ "moe",
49
+ "mamba",
50
+ "moe",
51
+ "mamba",
52
+ "attention",
53
+ "moe",
54
+ "mamba",
55
+ "moe",
56
+ "mamba",
57
+ "moe",
58
+ "mamba",
59
+ "attention",
60
+ "moe",
61
+ "mamba",
62
+ "moe",
63
+ "mamba",
64
+ "moe",
65
+ "mamba",
66
+ "moe",
67
+ "mamba",
68
+ "attention",
69
+ "moe",
70
+ "mamba",
71
+ "moe",
72
+ "mamba",
73
+ "moe",
74
+ "mamba",
75
+ "moe",
76
+ "mamba",
77
+ "attention",
78
+ "moe",
79
+ "mamba",
80
+ "moe",
81
+ "mamba",
82
+ "moe",
83
+ "mamba",
84
+ "attention",
85
+ "moe",
86
+ "mamba",
87
+ "moe",
88
+ "mamba",
89
+ "moe",
90
+ "mamba",
91
+ "moe",
92
+ "mamba",
93
+ "attention",
94
+ "moe",
95
+ "mamba",
96
+ "moe",
97
+ "mamba",
98
+ "moe",
99
+ "mamba",
100
+ "moe",
101
+ "mamba",
102
+ "attention",
103
+ "moe",
104
+ "mamba",
105
+ "moe",
106
+ "mamba",
107
+ "moe",
108
+ "mamba",
109
+ "attention",
110
+ "moe",
111
+ "mamba",
112
+ "moe",
113
+ "mamba",
114
+ "moe",
115
+ "mamba",
116
+ "moe",
117
+ "mamba",
118
+ "attention",
119
+ "moe",
120
+ "mamba",
121
+ "moe",
122
+ "mamba",
123
+ "moe",
124
+ "mamba",
125
+ "moe",
126
+ "mamba",
127
+ "moe"
128
+ ],
129
+ "mamba_head_dim": 64,
130
+ "mamba_hidden_act": "silu",
131
+ "mamba_num_heads": 256,
132
+ "mamba_proj_bias": false,
133
+ "mamba_ssm_cache_dtype": "float32",
134
+ "max_position_embeddings": 262144,
135
+ "mlp_bias": false,
136
+ "mlp_hidden_act": "relu2",
137
+ "model_type": "nemotron_h",
138
+ "moe_intermediate_size": 5120,
139
+ "moe_latent_size": 2048,
140
+ "moe_shared_expert_intermediate_size": 10240,
141
+ "moe_shared_expert_overlap": false,
142
+ "mtp_layers_block_type": [
143
+ "attention",
144
+ "moe"
145
+ ],
146
+ "n_group": 1,
147
+ "n_groups": 8,
148
+ "n_routed_experts": 512,
149
+ "n_shared_experts": 1,
150
+ "norm_eps": 1e-05,
151
+ "norm_topk_prob": true,
152
+ "num_attention_heads": 64,
153
+ "num_experts_per_tok": 22,
154
+ "num_key_value_heads": 2,
155
+ "num_logits_to_keep": 1,
156
+ "num_nextn_predict_layers": 1,
157
+ "pad_token_id": 0,
158
+ "partial_rotary_factor": 1.0,
159
+ "rescale_prenorm_residual": true,
160
+ "residual_in_fp32": false,
161
+ "rope_theta": 10000,
162
+ "routed_scaling_factor": 5.0,
163
+ "sliding_window": null,
164
+ "ssm_state_size": 128,
165
+ "tie_word_embeddings": false,
166
+ "time_step_floor": 0.0001,
167
+ "time_step_max": 0.1,
168
+ "time_step_min": 0.001,
169
+ "topk_group": 1,
170
+ "transformers_version": "4.57.6",
171
+ "use_bias": false,
172
+ "use_cache": true,
173
+ "use_conv_bias": true,
174
+ "use_mamba_kernels": true,
175
+ "vocab_size": 131072
176
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.57.6"
7
+ }
model-00008-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2332054f74c8c8d6a325bda5413c3d4088c42548ebeb3b1994ce1cf9b8601088
3
+ size 4991252600
model-00020-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d0709a536ee352d207eb226305ce7ee46919f51070f9403a3ba82becb922268
3
+ size 4991252664
model-00027-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0943a3012407c0ce99ecb11d5fb56d580127ccad663e96d5417837b73cdbd42f
3
+ size 4991252840
model-00039-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19aebb8ace54dab1f2d7a1133f330b12ff5a6b4811815ae2b694e6be8998a1bf
3
+ size 4991252840
model-00042-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b49426c203a0598f76aa54101643f372caee399a4f9633486d306f340a21129f
3
+ size 4987305872
model-00045-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66ea99f86c9e6099540cb97e3bade30a2f8c176cd210bb9507bf43172b0a999c
3
+ size 4991252840
model-00051-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9df9f1fe9422ed2321a1fe135feddfe742771b02d981bf051e475b0490677d3
3
+ size 4987305912
model-00053-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:068e62e33e2125cb050c98a45bdd14d44e3145d843e7061eaa8f5a1fbc003b8f
3
+ size 4991252840
model-00054-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:547fcedc811d6fad5252915dc5b36055abdbe968b3ac6080827a5402c2517731
3
+ size 4991252840
model-00055-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6bb0de613dd86e4ff1c029198ef8d194ca3a99fd8328023f41ed09cb87da97d7
3
+ size 4991252840
model-00067-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4268092e1ca5197215f00392877d92ba221f5c8ceef81bc68b0b373c8445ebba
3
+ size 4991252840
model-00071-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7d12c98827ca57e5ef74e2c88ae3fdb479a77e4ab2f5ed676d438388b471db6
3
+ size 4991252840
model-00074-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0585847fa62d37b424ebf77cdcb324938cf31d35103e40c0c10a64858c65242e
3
+ size 4987305872
model-00077-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f622efd103fbf29998504b096075a73620c30e4aa38d9572feaf64ce8ecb2711
3
+ size 4991252840
model-00084-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3335a9d5eeabeb77688b7132f05a52a826fbbeb5826fe499c6e39a280e23c7a1
3
+ size 4991252664
model-00089-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:332d3703db66c6fcedecba823944884ab29708033a3339a9b2179e5a505c7669
3
+ size 4991252776
model-00090-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d02e1b321be3f482baa5f1e224c4c1754c1678e5dc667d9b561f8462b052296
3
+ size 4991252840
model-00091-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ce2f54595e4acf23b9f7cd8cc2a5d320b67d7464432e631b3d5133b0aabaef3
3
+ size 4991252840
model-00092-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f9a342be47416c0c9a718ce9ba823fa586d92e23effad1ca635ab2707a32912f
3
+ size 4987305952
model-00094-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f562833e6cabe81e3c7ec78b3679399c3e5bfa73853627d43ee9d78282928533
3
+ size 4991252840
model-00100-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ecd732c2c0d05b7546d53760d44b1c6dbb7ec04ea6655ab2847a733655e9ced
3
+ size 4991252840
model-00102-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a55955b8af61ea534d71d1315def1bc1e4c188350845c2a28dd1a5b129c7557d
3
+ size 4995491376
model-00116-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8b6d514531967f22b083c951fc75b00f2809d192726b3857432a35b27eb5aeaf
3
+ size 4991252664
model-00117-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c3703eb66b7ed69c44c84ed098477f1c1c6cea42099871f1fb3079f8591076d
3
+ size 4991252840
model-00120-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8206d24f38de2ac7698dca778a8df1794d9998ea533542daf249a6c5d89ad6a
3
+ size 4987305808
model-00123-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84cb91c9e5143ed6f85ac62188c68e0cfae6ac846ebf5c7672f03e2da96429c5
3
+ size 4991252840
model-00130-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf0a9bfbbe7302b2f4104cac765f10dafa8cbe78c504d6e0fd53b729f121fa52
3
+ size 4991252736
model-00132-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ed48fc4384ae863d3cb089f139361fcd659c0ba6776a899bdd2b3bbccc5e9e3
3
+ size 4991252840
model-00137-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aac67bdc9eee3b355bff599f116831750c639aa54c006fa00978f0a68bf88a87
3
+ size 4991252840
model-00140-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ba271b2f680ceffd1ce050943f08d893941f5386110bf6a23cb2ff0d226c439
3
+ size 4991252840
model-00143-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:499af916cf980de8ec54b3e5c4f4acf1d9e00858cfc4d18b4033515427a8b457
3
+ size 4987305824
model-00146-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb6e7bbcbc871c57ae57902ffb841c7f47144dd5a80b3967fb1698ecdda70c74
3
+ size 4991252840
model-00154-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abf58aa54d88de553ff6fc72ee8c35e35ab61835dde3fd8df0ed98503caf0e0d
3
+ size 4991252840
model-00165-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13d9ea998323e1549c4e6986de08937df3e9a354f3f9b6b525e8fbe5002d475e
3
+ size 4668492520
model-00168-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a49703e503910ac2819369329aebff52627b27a4a8d20a12163bd01242506f6a
3
+ size 4991252840
model-00175-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76eef5efb8f3114c8895744c0beaec8757a0a6a295903c83370eb8eeebc00245
3
+ size 4987305816
model-00187-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d95cd8fe6fc63db4bfb172b470ab00803fb6ecfa8dff4c87d8bbba39e68ec46
3
+ size 4991252840
model-00191-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4089eb54935f2c387fa219dede60026521e0177794553e90d72ca35f00588dd
3
+ size 4991252840
model-00200-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac0ab09f612e8490ed8c0f5abc502373f25603b93eb06e805d69e442a317abd7
3
+ size 4991252840
model-00202-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f250594afbe694d524d6750fc743b115b6d16122e20002928be1dc04c94d6eb5
3
+ size 4987305976
model-00203-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93dd754e79ec0f8883f60eedfe67edb8bfaf4e390b8238abbeb5f993c2306efc
3
+ size 4991252928
model-00205-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5ab7e351b3e502b23dba7c28a254676ca20d845941b528c7898c71fd071ba86
3
+ size 4991253080
model-00214-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:430d6dbed5f225c7a60d951f374d9da2030ae8fc435b18f718539a3f17cddcb5
3
+ size 4991253080
model-00217-of-00225.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fd7183b3bf879196263ce646f4ff77d8913ce4cb92e5c13c04576f05e745792
3
+ size 4991253008
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|im_end|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<|im_end|>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff