diff --git "a/decoder_step.mlmodelc/model.mil" "b/decoder_step.mlmodelc/model.mil" --- "a/decoder_step.mlmodelc/model.mil" +++ "b/decoder_step.mlmodelc/model.mil" @@ -6,19 +6,19 @@ program(1.0) tensor var_502_validate_indices_0 = const()[name = tensor("op_502_validate_indices_0"), val = tensor(false)]; tensor dec_position_embeddings_weight_to_fp16_quantized = constexpr_affine_dequantize()[axis = tensor(0), name = tensor("dec_position_embeddings_weight_to_fp16_quantized"), quantized_data = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64))), scale = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1575104))), zero_point = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1572992)))]; tensor position_to_int16_dtype_0 = const()[name = tensor("position_to_int16_dtype_0"), val = tensor("int16")]; - tensor cast_89_dtype_0 = const()[name = tensor("cast_89_dtype_0"), val = tensor("int32")]; + tensor cast_63_dtype_0 = const()[name = tensor("cast_63_dtype_0"), val = tensor("int32")]; tensor greater_equal_0_y_0 = const()[name = tensor("greater_equal_0_y_0"), val = tensor(0)]; - tensor position_to_int16 = cast(dtype = position_to_int16_dtype_0, x = position)[name = tensor("cast_57")]; - tensor cast_89 = cast(dtype = cast_89_dtype_0, x = position_to_int16)[name = tensor("cast_56")]; - tensor greater_equal_0 = greater_equal(x = cast_89, y = greater_equal_0_y_0)[name = tensor("greater_equal_0")]; + tensor position_to_int16 = cast(dtype = position_to_int16_dtype_0, x = position)[name = tensor("cast_83")]; + tensor cast_63 = cast(dtype = cast_63_dtype_0, x = position_to_int16)[name = tensor("cast_82")]; + tensor greater_equal_0 = greater_equal(x = cast_63, y = greater_equal_0_y_0)[name = tensor("greater_equal_0")]; tensor slice_by_index_0 = const()[name = tensor("slice_by_index_0"), val = tensor(2048)]; - tensor add_0 = add(x = cast_89, y = slice_by_index_0)[name = tensor("add_0")]; - tensor select_0 = select(a = cast_89, b = add_0, cond = greater_equal_0)[name = tensor("select_0")]; + tensor add_0 = add(x = cast_63, y = slice_by_index_0)[name = tensor("add_0")]; + tensor select_0 = select(a = cast_63, b = add_0, cond = greater_equal_0)[name = tensor("select_0")]; tensor select_0_to_int16_dtype_0 = const()[name = tensor("select_0_to_int16_dtype_0"), val = tensor("int16")]; tensor cast_0_dtype_0 = const()[name = tensor("cast_0_dtype_0"), val = tensor("int32")]; tensor greater_equal_0_y_0_1 = const()[name = tensor("greater_equal_0_y_0_1"), val = tensor(0)]; - tensor select_0_to_int16 = cast(dtype = select_0_to_int16_dtype_0, x = select_0)[name = tensor("cast_55")]; - tensor cast_0 = cast(dtype = cast_0_dtype_0, x = select_0_to_int16)[name = tensor("cast_54")]; + tensor select_0_to_int16 = cast(dtype = select_0_to_int16_dtype_0, x = select_0)[name = tensor("cast_81")]; + tensor cast_0 = cast(dtype = cast_0_dtype_0, x = select_0_to_int16)[name = tensor("cast_80")]; tensor greater_equal_0_1 = greater_equal(x = cast_0, y = greater_equal_0_y_0_1)[name = tensor("greater_equal_0_1")]; tensor slice_by_index_0_1 = const()[name = tensor("slice_by_index_0_1"), val = tensor(2048)]; tensor add_0_1 = add(x = cast_0, y = slice_by_index_0_1)[name = tensor("add_0_1")]; @@ -26,15 +26,15 @@ program(1.0) tensor op_502_cast_fp16_cast_uint16_cast_uint16_axis_0 = const()[name = tensor("op_502_cast_fp16_cast_uint16_cast_uint16_axis_0"), val = tensor(0)]; tensor op_502_cast_fp16_cast_uint16_cast_uint16 = gather(axis = op_502_cast_fp16_cast_uint16_cast_uint16_axis_0, batch_dims = var_502_batch_dims_0, indices = select_0_1, validate_indices = var_502_validate_indices_0, x = dec_position_embeddings_weight_to_fp16_quantized)[name = tensor("op_502_cast_fp16_cast_uint16_cast_uint16")]; tensor audio_emb_to_fp16_dtype_0 = const()[name = tensor("audio_emb_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor audio_emb_to_fp16 = cast(dtype = audio_emb_to_fp16_dtype_0, x = audio_emb)[name = tensor("cast_53")]; + tensor audio_emb_to_fp16 = cast(dtype = audio_emb_to_fp16_dtype_0, x = audio_emb)[name = tensor("cast_79")]; tensor input_3_cast_fp16 = add(x = audio_emb_to_fp16, y = op_502_cast_fp16_cast_uint16_cast_uint16)[name = tensor("input_3_cast_fp16")]; tensor idx_range_promoted_to_fp16 = const()[name = tensor("idx_range_promoted_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1579264)))]; tensor var_515_to_fp16_dtype_0 = const()[name = tensor("op_515_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor position_to_fp16 = cast(dtype = var_515_to_fp16_dtype_0, x = position)[name = tensor("cast_52")]; + tensor position_to_fp16 = cast(dtype = var_515_to_fp16_dtype_0, x = position)[name = tensor("cast_78")]; tensor var_516_cast_fp16 = less_equal(x = idx_range_promoted_to_fp16, y = position_to_fp16)[name = tensor("op_516_cast_fp16")]; tensor sa_key_mask_axes_0 = const()[name = tensor("sa_key_mask_axes_0"), val = tensor([0])]; tensor sa_key_mask_1_to_fp16_dtype_0 = const()[name = tensor("sa_key_mask_1_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor var_516_cast_fp16_to_fp16 = cast(dtype = sa_key_mask_1_to_fp16_dtype_0, x = var_516_cast_fp16)[name = tensor("cast_51")]; + tensor var_516_cast_fp16_to_fp16 = cast(dtype = sa_key_mask_1_to_fp16_dtype_0, x = var_516_cast_fp16)[name = tensor("cast_77")]; tensor sa_key_mask_cast_fp16 = expand_dims(axes = sa_key_mask_axes_0, x = var_516_cast_fp16_to_fp16)[name = tensor("sa_key_mask_cast_fp16")]; tensor input_5_axes_0 = const()[name = tensor("input_5_axes_0"), val = tensor([-1])]; tensor dec_layers_0_norm_self_weight_to_fp16 = const()[name = tensor("dec_layers_0_norm_self_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1580544)))]; @@ -63,26 +63,28 @@ program(1.0) tensor var_585_cast_fp16 = equal(x = idx_range_promoted_to_fp16, y = position_to_fp16)[name = tensor("op_585_cast_fp16")]; tensor var_595 = const()[name = tensor("op_595"), val = tensor([1, 600, 1, 1])]; tensor write_oh_1_to_fp16_dtype_0 = const()[name = tensor("write_oh_1_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor var_585_cast_fp16_to_fp16 = cast(dtype = write_oh_1_to_fp16_dtype_0, x = var_585_cast_fp16)[name = tensor("cast_50")]; + tensor var_585_cast_fp16_to_fp16 = cast(dtype = write_oh_1_to_fp16_dtype_0, x = var_585_cast_fp16)[name = tensor("cast_76")]; tensor write_oh_b_1_cast_fp16 = reshape(shape = var_595, x = var_585_cast_fp16_to_fp16)[name = tensor("write_oh_b_1_cast_fp16")]; tensor var_597_to_fp16 = const()[name = tensor("op_597_to_fp16"), val = tensor(0x1p+0)]; tensor var_599_cast_fp16 = sub(x = var_597_to_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_599_cast_fp16")]; tensor sa_k_in_0_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_0_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_0_to_fp16 = cast(dtype = sa_k_in_0_to_fp16_dtype_0, x = sa_k_in_0)[name = tensor("cast_49")]; + tensor sa_k_in_0_to_fp16 = cast(dtype = sa_k_in_0_to_fp16_dtype_0, x = sa_k_in_0)[name = tensor("cast_75")]; tensor var_600_cast_fp16 = mul(x = sa_k_in_0_to_fp16, y = var_599_cast_fp16)[name = tensor("op_600_cast_fp16")]; tensor var_601_cast_fp16 = mul(x = new_k_1_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_601_cast_fp16")]; - tensor sa_k_out_0 = add(x = var_600_cast_fp16, y = var_601_cast_fp16)[name = tensor("sa_k_out_1_cast_fp16")]; + tensor sa_k_out_1_cast_fp16 = add(x = var_600_cast_fp16, y = var_601_cast_fp16)[name = tensor("sa_k_out_1_cast_fp16")]; + tensor sa_k_out_1_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_1_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_0_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_0_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_0_to_fp16 = cast(dtype = sa_v_in_0_to_fp16_dtype_0, x = sa_v_in_0)[name = tensor("cast_48")]; + tensor sa_v_in_0_to_fp16 = cast(dtype = sa_v_in_0_to_fp16_dtype_0, x = sa_v_in_0)[name = tensor("cast_74")]; tensor var_607_cast_fp16 = mul(x = sa_v_in_0_to_fp16, y = var_599_cast_fp16)[name = tensor("op_607_cast_fp16")]; tensor var_608_cast_fp16 = mul(x = new_v_1_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_608_cast_fp16")]; - tensor sa_v_out_0 = add(x = var_607_cast_fp16, y = var_608_cast_fp16)[name = tensor("sa_v_out_1_cast_fp16")]; + tensor sa_v_out_1_cast_fp16 = add(x = var_607_cast_fp16, y = var_608_cast_fp16)[name = tensor("sa_v_out_1_cast_fp16")]; + tensor sa_v_out_1_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_1_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_627 = const()[name = tensor("op_627"), val = tensor([0, 2, -3, -1])]; tensor var_629_transpose_x_0 = const()[name = tensor("op_629_transpose_x_0"), val = tensor(false)]; tensor var_629_transpose_y_0 = const()[name = tensor("op_629_transpose_y_0"), val = tensor(false)]; tensor transpose_96_perm_0 = const()[name = tensor("transpose_96_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_97_perm_0 = const()[name = tensor("transpose_97_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_97 = transpose(perm = transpose_97_perm_0, x = sa_k_out_0)[name = tensor("transpose_262")]; + tensor transpose_97 = transpose(perm = transpose_97_perm_0, x = sa_k_out_1_cast_fp16)[name = tensor("transpose_262")]; tensor transpose_96 = transpose(perm = transpose_96_perm_0, x = q_1_cast_fp16)[name = tensor("transpose_263")]; tensor var_629_cast_fp16 = matmul(transpose_x = var_629_transpose_x_0, transpose_y = var_629_transpose_y_0, x = transpose_96, y = transpose_97)[name = tensor("op_629_cast_fp16")]; tensor var_630_to_fp16 = const()[name = tensor("op_630_to_fp16"), val = tensor(0x1p-3)]; @@ -99,7 +101,7 @@ program(1.0) tensor probs_1_cast_fp16 = softmax(axis = var_650, x = scores_3_cast_fp16)[name = tensor("probs_1_cast_fp16")]; tensor var_653_transpose_x_0 = const()[name = tensor("op_653_transpose_x_0"), val = tensor(false)]; tensor var_653_transpose_y_0 = const()[name = tensor("op_653_transpose_y_0"), val = tensor(false)]; - tensor v_t_1_cast_fp16 = transpose(perm = var_627, x = sa_v_out_0)[name = tensor("transpose_261")]; + tensor v_t_1_cast_fp16 = transpose(perm = var_627, x = sa_v_out_1_cast_fp16)[name = tensor("transpose_261")]; tensor var_653_cast_fp16 = matmul(transpose_x = var_653_transpose_x_0, transpose_y = var_653_transpose_y_0, x = probs_1_cast_fp16, y = v_t_1_cast_fp16)[name = tensor("op_653_cast_fp16")]; tensor var_658 = const()[name = tensor("op_658"), val = tensor([0, 2, 1, 3])]; tensor var_663 = const()[name = tensor("op_663"), val = tensor([1, 1, -1])]; @@ -125,7 +127,7 @@ program(1.0) tensor xa_k_0_to_fp16_dtype_0 = const()[name = tensor("xa_k_0_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_98_perm_0 = const()[name = tensor("transpose_98_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_99_perm_0 = const()[name = tensor("transpose_99_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_0_to_fp16 = cast(dtype = xa_k_0_to_fp16_dtype_0, x = xa_k_0)[name = tensor("cast_47")]; + tensor xa_k_0_to_fp16 = cast(dtype = xa_k_0_to_fp16_dtype_0, x = xa_k_0)[name = tensor("cast_73")]; tensor transpose_99 = transpose(perm = transpose_99_perm_0, x = xa_k_0_to_fp16)[name = tensor("transpose_258")]; tensor transpose_98 = transpose(perm = transpose_98_perm_0, x = xq_proj_1_cast_fp16)[name = tensor("transpose_259")]; tensor var_704_cast_fp16 = matmul(transpose_x = var_704_transpose_x_0, transpose_y = var_704_transpose_y_0, x = transpose_98, y = transpose_99)[name = tensor("op_704_cast_fp16")]; @@ -133,7 +135,7 @@ program(1.0) tensor xscores_1_cast_fp16 = mul(x = var_704_cast_fp16, y = var_705_to_fp16)[name = tensor("xscores_1_cast_fp16")]; tensor var_713_axes_0 = const()[name = tensor("op_713_axes_0"), val = tensor([1])]; tensor encoder_mask_to_fp16_dtype_0 = const()[name = tensor("encoder_mask_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor encoder_mask_to_fp16 = cast(dtype = encoder_mask_to_fp16_dtype_0, x = encoder_mask)[name = tensor("cast_46")]; + tensor encoder_mask_to_fp16 = cast(dtype = encoder_mask_to_fp16_dtype_0, x = encoder_mask)[name = tensor("cast_72")]; tensor var_713_cast_fp16 = expand_dims(axes = var_713_axes_0, x = encoder_mask_to_fp16)[name = tensor("op_713_cast_fp16")]; tensor var_715_axes_0 = const()[name = tensor("op_715_axes_0"), val = tensor([2])]; tensor var_715_cast_fp16 = expand_dims(axes = var_715_axes_0, x = var_713_cast_fp16)[name = tensor("op_715_cast_fp16")]; @@ -145,7 +147,7 @@ program(1.0) tensor xprobs_1_cast_fp16 = softmax(axis = var_725, x = xscores_3_cast_fp16)[name = tensor("xprobs_1_cast_fp16")]; tensor var_728_transpose_x_0 = const()[name = tensor("op_728_transpose_x_0"), val = tensor(false)]; tensor var_728_transpose_y_0 = const()[name = tensor("op_728_transpose_y_0"), val = tensor(false)]; - tensor xa_v_0_to_fp16 = cast(dtype = xa_v_0_to_fp16_dtype_0, x = xa_v_0)[name = tensor("cast_45")]; + tensor xa_v_0_to_fp16 = cast(dtype = xa_v_0_to_fp16_dtype_0, x = xa_v_0)[name = tensor("cast_71")]; tensor xvT_1_cast_fp16 = transpose(perm = var_702, x = xa_v_0_to_fp16)[name = tensor("transpose_257")]; tensor var_728_cast_fp16 = matmul(transpose_x = var_728_transpose_x_0, transpose_y = var_728_transpose_y_0, x = xprobs_1_cast_fp16, y = xvT_1_cast_fp16)[name = tensor("op_728_cast_fp16")]; tensor var_733 = const()[name = tensor("op_733"), val = tensor([0, 2, 1, 3])]; @@ -204,21 +206,23 @@ program(1.0) tensor new_v_3_squeeze_mask_0 = const()[name = tensor("new_v_3_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_3_cast_fp16 = slice_by_index(begin = new_v_3_begin_0, end = new_v_3_end_0, end_mask = new_v_3_end_mask_0, squeeze_mask = new_v_3_squeeze_mask_0, x = qkv_7_cast_fp16)[name = tensor("new_v_3_cast_fp16")]; tensor sa_k_in_1_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_1_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_1_to_fp16 = cast(dtype = sa_k_in_1_to_fp16_dtype_0, x = sa_k_in_1)[name = tensor("cast_44")]; + tensor sa_k_in_1_to_fp16 = cast(dtype = sa_k_in_1_to_fp16_dtype_0, x = sa_k_in_1)[name = tensor("cast_70")]; tensor var_860_cast_fp16 = mul(x = sa_k_in_1_to_fp16, y = var_599_cast_fp16)[name = tensor("op_860_cast_fp16")]; tensor var_861_cast_fp16 = mul(x = new_k_3_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_861_cast_fp16")]; - tensor sa_k_out_1 = add(x = var_860_cast_fp16, y = var_861_cast_fp16)[name = tensor("sa_k_out_3_cast_fp16")]; + tensor sa_k_out_3_cast_fp16 = add(x = var_860_cast_fp16, y = var_861_cast_fp16)[name = tensor("sa_k_out_3_cast_fp16")]; + tensor sa_k_out_3_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_3_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_1_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_1_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_1_to_fp16 = cast(dtype = sa_v_in_1_to_fp16_dtype_0, x = sa_v_in_1)[name = tensor("cast_43")]; + tensor sa_v_in_1_to_fp16 = cast(dtype = sa_v_in_1_to_fp16_dtype_0, x = sa_v_in_1)[name = tensor("cast_69")]; tensor var_867_cast_fp16 = mul(x = sa_v_in_1_to_fp16, y = var_599_cast_fp16)[name = tensor("op_867_cast_fp16")]; tensor var_868_cast_fp16 = mul(x = new_v_3_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_868_cast_fp16")]; - tensor sa_v_out_1 = add(x = var_867_cast_fp16, y = var_868_cast_fp16)[name = tensor("sa_v_out_3_cast_fp16")]; + tensor sa_v_out_3_cast_fp16 = add(x = var_867_cast_fp16, y = var_868_cast_fp16)[name = tensor("sa_v_out_3_cast_fp16")]; + tensor sa_v_out_3_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_3_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_887 = const()[name = tensor("op_887"), val = tensor([0, 2, -3, -1])]; tensor var_889_transpose_x_0 = const()[name = tensor("op_889_transpose_x_0"), val = tensor(false)]; tensor var_889_transpose_y_0 = const()[name = tensor("op_889_transpose_y_0"), val = tensor(false)]; tensor transpose_100_perm_0 = const()[name = tensor("transpose_100_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_101_perm_0 = const()[name = tensor("transpose_101_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_101 = transpose(perm = transpose_101_perm_0, x = sa_k_out_1)[name = tensor("transpose_252")]; + tensor transpose_101 = transpose(perm = transpose_101_perm_0, x = sa_k_out_3_cast_fp16)[name = tensor("transpose_252")]; tensor transpose_100 = transpose(perm = transpose_100_perm_0, x = q_3_cast_fp16)[name = tensor("transpose_253")]; tensor var_889_cast_fp16 = matmul(transpose_x = var_889_transpose_x_0, transpose_y = var_889_transpose_y_0, x = transpose_100, y = transpose_101)[name = tensor("op_889_cast_fp16")]; tensor var_890_to_fp16 = const()[name = tensor("op_890_to_fp16"), val = tensor(0x1p-3)]; @@ -229,7 +233,7 @@ program(1.0) tensor probs_3_cast_fp16 = softmax(axis = var_910, x = scores_7_cast_fp16)[name = tensor("probs_3_cast_fp16")]; tensor var_913_transpose_x_0 = const()[name = tensor("op_913_transpose_x_0"), val = tensor(false)]; tensor var_913_transpose_y_0 = const()[name = tensor("op_913_transpose_y_0"), val = tensor(false)]; - tensor v_t_3_cast_fp16 = transpose(perm = var_887, x = sa_v_out_1)[name = tensor("transpose_251")]; + tensor v_t_3_cast_fp16 = transpose(perm = var_887, x = sa_v_out_3_cast_fp16)[name = tensor("transpose_251")]; tensor var_913_cast_fp16 = matmul(transpose_x = var_913_transpose_x_0, transpose_y = var_913_transpose_y_0, x = probs_3_cast_fp16, y = v_t_3_cast_fp16)[name = tensor("op_913_cast_fp16")]; tensor var_918 = const()[name = tensor("op_918"), val = tensor([0, 2, 1, 3])]; tensor var_923 = const()[name = tensor("op_923"), val = tensor([1, 1, -1])]; @@ -253,7 +257,7 @@ program(1.0) tensor xa_k_1_to_fp16_dtype_0 = const()[name = tensor("xa_k_1_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_102_perm_0 = const()[name = tensor("transpose_102_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_103_perm_0 = const()[name = tensor("transpose_103_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_1_to_fp16 = cast(dtype = xa_k_1_to_fp16_dtype_0, x = xa_k_1)[name = tensor("cast_42")]; + tensor xa_k_1_to_fp16 = cast(dtype = xa_k_1_to_fp16_dtype_0, x = xa_k_1)[name = tensor("cast_68")]; tensor transpose_103 = transpose(perm = transpose_103_perm_0, x = xa_k_1_to_fp16)[name = tensor("transpose_248")]; tensor transpose_102 = transpose(perm = transpose_102_perm_0, x = xq_proj_3_cast_fp16)[name = tensor("transpose_249")]; tensor var_964_cast_fp16 = matmul(transpose_x = var_964_transpose_x_0, transpose_y = var_964_transpose_y_0, x = transpose_102, y = transpose_103)[name = tensor("op_964_cast_fp16")]; @@ -265,7 +269,7 @@ program(1.0) tensor xprobs_3_cast_fp16 = softmax(axis = var_985, x = xscores_7_cast_fp16)[name = tensor("xprobs_3_cast_fp16")]; tensor var_988_transpose_x_0 = const()[name = tensor("op_988_transpose_x_0"), val = tensor(false)]; tensor var_988_transpose_y_0 = const()[name = tensor("op_988_transpose_y_0"), val = tensor(false)]; - tensor xa_v_1_to_fp16 = cast(dtype = xa_v_1_to_fp16_dtype_0, x = xa_v_1)[name = tensor("cast_41")]; + tensor xa_v_1_to_fp16 = cast(dtype = xa_v_1_to_fp16_dtype_0, x = xa_v_1)[name = tensor("cast_67")]; tensor xvT_3_cast_fp16 = transpose(perm = var_962, x = xa_v_1_to_fp16)[name = tensor("transpose_247")]; tensor var_988_cast_fp16 = matmul(transpose_x = var_988_transpose_x_0, transpose_y = var_988_transpose_y_0, x = xprobs_3_cast_fp16, y = xvT_3_cast_fp16)[name = tensor("op_988_cast_fp16")]; tensor var_993 = const()[name = tensor("op_993"), val = tensor([0, 2, 1, 3])]; @@ -324,21 +328,23 @@ program(1.0) tensor new_v_5_squeeze_mask_0 = const()[name = tensor("new_v_5_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_5_cast_fp16 = slice_by_index(begin = new_v_5_begin_0, end = new_v_5_end_0, end_mask = new_v_5_end_mask_0, squeeze_mask = new_v_5_squeeze_mask_0, x = qkv_11_cast_fp16)[name = tensor("new_v_5_cast_fp16")]; tensor sa_k_in_2_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_2_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_2_to_fp16 = cast(dtype = sa_k_in_2_to_fp16_dtype_0, x = sa_k_in_2)[name = tensor("cast_40")]; + tensor sa_k_in_2_to_fp16 = cast(dtype = sa_k_in_2_to_fp16_dtype_0, x = sa_k_in_2)[name = tensor("cast_66")]; tensor var_1120_cast_fp16 = mul(x = sa_k_in_2_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1120_cast_fp16")]; tensor var_1121_cast_fp16 = mul(x = new_k_5_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1121_cast_fp16")]; - tensor sa_k_out_2 = add(x = var_1120_cast_fp16, y = var_1121_cast_fp16)[name = tensor("sa_k_out_5_cast_fp16")]; + tensor sa_k_out_5_cast_fp16 = add(x = var_1120_cast_fp16, y = var_1121_cast_fp16)[name = tensor("sa_k_out_5_cast_fp16")]; + tensor sa_k_out_5_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_5_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_2_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_2_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_2_to_fp16 = cast(dtype = sa_v_in_2_to_fp16_dtype_0, x = sa_v_in_2)[name = tensor("cast_39")]; + tensor sa_v_in_2_to_fp16 = cast(dtype = sa_v_in_2_to_fp16_dtype_0, x = sa_v_in_2)[name = tensor("cast_65")]; tensor var_1127_cast_fp16 = mul(x = sa_v_in_2_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1127_cast_fp16")]; tensor var_1128_cast_fp16 = mul(x = new_v_5_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1128_cast_fp16")]; - tensor sa_v_out_2 = add(x = var_1127_cast_fp16, y = var_1128_cast_fp16)[name = tensor("sa_v_out_5_cast_fp16")]; + tensor sa_v_out_5_cast_fp16 = add(x = var_1127_cast_fp16, y = var_1128_cast_fp16)[name = tensor("sa_v_out_5_cast_fp16")]; + tensor sa_v_out_5_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_5_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_1147 = const()[name = tensor("op_1147"), val = tensor([0, 2, -3, -1])]; tensor var_1149_transpose_x_0 = const()[name = tensor("op_1149_transpose_x_0"), val = tensor(false)]; tensor var_1149_transpose_y_0 = const()[name = tensor("op_1149_transpose_y_0"), val = tensor(false)]; tensor transpose_104_perm_0 = const()[name = tensor("transpose_104_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_105_perm_0 = const()[name = tensor("transpose_105_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_105 = transpose(perm = transpose_105_perm_0, x = sa_k_out_2)[name = tensor("transpose_242")]; + tensor transpose_105 = transpose(perm = transpose_105_perm_0, x = sa_k_out_5_cast_fp16)[name = tensor("transpose_242")]; tensor transpose_104 = transpose(perm = transpose_104_perm_0, x = q_5_cast_fp16)[name = tensor("transpose_243")]; tensor var_1149_cast_fp16 = matmul(transpose_x = var_1149_transpose_x_0, transpose_y = var_1149_transpose_y_0, x = transpose_104, y = transpose_105)[name = tensor("op_1149_cast_fp16")]; tensor var_1150_to_fp16 = const()[name = tensor("op_1150_to_fp16"), val = tensor(0x1p-3)]; @@ -349,7 +355,7 @@ program(1.0) tensor probs_5_cast_fp16 = softmax(axis = var_1170, x = scores_11_cast_fp16)[name = tensor("probs_5_cast_fp16")]; tensor var_1173_transpose_x_0 = const()[name = tensor("op_1173_transpose_x_0"), val = tensor(false)]; tensor var_1173_transpose_y_0 = const()[name = tensor("op_1173_transpose_y_0"), val = tensor(false)]; - tensor v_t_5_cast_fp16 = transpose(perm = var_1147, x = sa_v_out_2)[name = tensor("transpose_241")]; + tensor v_t_5_cast_fp16 = transpose(perm = var_1147, x = sa_v_out_5_cast_fp16)[name = tensor("transpose_241")]; tensor var_1173_cast_fp16 = matmul(transpose_x = var_1173_transpose_x_0, transpose_y = var_1173_transpose_y_0, x = probs_5_cast_fp16, y = v_t_5_cast_fp16)[name = tensor("op_1173_cast_fp16")]; tensor var_1178 = const()[name = tensor("op_1178"), val = tensor([0, 2, 1, 3])]; tensor var_1183 = const()[name = tensor("op_1183"), val = tensor([1, 1, -1])]; @@ -373,7 +379,7 @@ program(1.0) tensor xa_k_2_to_fp16_dtype_0 = const()[name = tensor("xa_k_2_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_106_perm_0 = const()[name = tensor("transpose_106_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_107_perm_0 = const()[name = tensor("transpose_107_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_2_to_fp16 = cast(dtype = xa_k_2_to_fp16_dtype_0, x = xa_k_2)[name = tensor("cast_38")]; + tensor xa_k_2_to_fp16 = cast(dtype = xa_k_2_to_fp16_dtype_0, x = xa_k_2)[name = tensor("cast_64")]; tensor transpose_107 = transpose(perm = transpose_107_perm_0, x = xa_k_2_to_fp16)[name = tensor("transpose_238")]; tensor transpose_106 = transpose(perm = transpose_106_perm_0, x = xq_proj_5_cast_fp16)[name = tensor("transpose_239")]; tensor var_1224_cast_fp16 = matmul(transpose_x = var_1224_transpose_x_0, transpose_y = var_1224_transpose_y_0, x = transpose_106, y = transpose_107)[name = tensor("op_1224_cast_fp16")]; @@ -385,7 +391,7 @@ program(1.0) tensor xprobs_5_cast_fp16 = softmax(axis = var_1245, x = xscores_11_cast_fp16)[name = tensor("xprobs_5_cast_fp16")]; tensor var_1248_transpose_x_0 = const()[name = tensor("op_1248_transpose_x_0"), val = tensor(false)]; tensor var_1248_transpose_y_0 = const()[name = tensor("op_1248_transpose_y_0"), val = tensor(false)]; - tensor xa_v_2_to_fp16 = cast(dtype = xa_v_2_to_fp16_dtype_0, x = xa_v_2)[name = tensor("cast_37")]; + tensor xa_v_2_to_fp16 = cast(dtype = xa_v_2_to_fp16_dtype_0, x = xa_v_2)[name = tensor("cast_63")]; tensor xvT_5_cast_fp16 = transpose(perm = var_1222, x = xa_v_2_to_fp16)[name = tensor("transpose_237")]; tensor var_1248_cast_fp16 = matmul(transpose_x = var_1248_transpose_x_0, transpose_y = var_1248_transpose_y_0, x = xprobs_5_cast_fp16, y = xvT_5_cast_fp16)[name = tensor("op_1248_cast_fp16")]; tensor var_1253 = const()[name = tensor("op_1253"), val = tensor([0, 2, 1, 3])]; @@ -444,21 +450,23 @@ program(1.0) tensor new_v_7_squeeze_mask_0 = const()[name = tensor("new_v_7_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_7_cast_fp16 = slice_by_index(begin = new_v_7_begin_0, end = new_v_7_end_0, end_mask = new_v_7_end_mask_0, squeeze_mask = new_v_7_squeeze_mask_0, x = qkv_15_cast_fp16)[name = tensor("new_v_7_cast_fp16")]; tensor sa_k_in_3_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_3_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_3_to_fp16 = cast(dtype = sa_k_in_3_to_fp16_dtype_0, x = sa_k_in_3)[name = tensor("cast_36")]; + tensor sa_k_in_3_to_fp16 = cast(dtype = sa_k_in_3_to_fp16_dtype_0, x = sa_k_in_3)[name = tensor("cast_62")]; tensor var_1380_cast_fp16 = mul(x = sa_k_in_3_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1380_cast_fp16")]; tensor var_1381_cast_fp16 = mul(x = new_k_7_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1381_cast_fp16")]; - tensor sa_k_out_3 = add(x = var_1380_cast_fp16, y = var_1381_cast_fp16)[name = tensor("sa_k_out_7_cast_fp16")]; + tensor sa_k_out_7_cast_fp16 = add(x = var_1380_cast_fp16, y = var_1381_cast_fp16)[name = tensor("sa_k_out_7_cast_fp16")]; + tensor sa_k_out_7_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_7_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_3_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_3_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_3_to_fp16 = cast(dtype = sa_v_in_3_to_fp16_dtype_0, x = sa_v_in_3)[name = tensor("cast_35")]; + tensor sa_v_in_3_to_fp16 = cast(dtype = sa_v_in_3_to_fp16_dtype_0, x = sa_v_in_3)[name = tensor("cast_61")]; tensor var_1387_cast_fp16 = mul(x = sa_v_in_3_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1387_cast_fp16")]; tensor var_1388_cast_fp16 = mul(x = new_v_7_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1388_cast_fp16")]; - tensor sa_v_out_3 = add(x = var_1387_cast_fp16, y = var_1388_cast_fp16)[name = tensor("sa_v_out_7_cast_fp16")]; + tensor sa_v_out_7_cast_fp16 = add(x = var_1387_cast_fp16, y = var_1388_cast_fp16)[name = tensor("sa_v_out_7_cast_fp16")]; + tensor sa_v_out_7_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_7_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_1407 = const()[name = tensor("op_1407"), val = tensor([0, 2, -3, -1])]; tensor var_1409_transpose_x_0 = const()[name = tensor("op_1409_transpose_x_0"), val = tensor(false)]; tensor var_1409_transpose_y_0 = const()[name = tensor("op_1409_transpose_y_0"), val = tensor(false)]; tensor transpose_108_perm_0 = const()[name = tensor("transpose_108_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_109_perm_0 = const()[name = tensor("transpose_109_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_109 = transpose(perm = transpose_109_perm_0, x = sa_k_out_3)[name = tensor("transpose_232")]; + tensor transpose_109 = transpose(perm = transpose_109_perm_0, x = sa_k_out_7_cast_fp16)[name = tensor("transpose_232")]; tensor transpose_108 = transpose(perm = transpose_108_perm_0, x = q_7_cast_fp16)[name = tensor("transpose_233")]; tensor var_1409_cast_fp16 = matmul(transpose_x = var_1409_transpose_x_0, transpose_y = var_1409_transpose_y_0, x = transpose_108, y = transpose_109)[name = tensor("op_1409_cast_fp16")]; tensor var_1410_to_fp16 = const()[name = tensor("op_1410_to_fp16"), val = tensor(0x1p-3)]; @@ -469,7 +477,7 @@ program(1.0) tensor probs_7_cast_fp16 = softmax(axis = var_1430, x = scores_15_cast_fp16)[name = tensor("probs_7_cast_fp16")]; tensor var_1433_transpose_x_0 = const()[name = tensor("op_1433_transpose_x_0"), val = tensor(false)]; tensor var_1433_transpose_y_0 = const()[name = tensor("op_1433_transpose_y_0"), val = tensor(false)]; - tensor v_t_7_cast_fp16 = transpose(perm = var_1407, x = sa_v_out_3)[name = tensor("transpose_231")]; + tensor v_t_7_cast_fp16 = transpose(perm = var_1407, x = sa_v_out_7_cast_fp16)[name = tensor("transpose_231")]; tensor var_1433_cast_fp16 = matmul(transpose_x = var_1433_transpose_x_0, transpose_y = var_1433_transpose_y_0, x = probs_7_cast_fp16, y = v_t_7_cast_fp16)[name = tensor("op_1433_cast_fp16")]; tensor var_1438 = const()[name = tensor("op_1438"), val = tensor([0, 2, 1, 3])]; tensor var_1443 = const()[name = tensor("op_1443"), val = tensor([1, 1, -1])]; @@ -493,7 +501,7 @@ program(1.0) tensor xa_k_3_to_fp16_dtype_0 = const()[name = tensor("xa_k_3_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_110_perm_0 = const()[name = tensor("transpose_110_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_111_perm_0 = const()[name = tensor("transpose_111_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_3_to_fp16 = cast(dtype = xa_k_3_to_fp16_dtype_0, x = xa_k_3)[name = tensor("cast_34")]; + tensor xa_k_3_to_fp16 = cast(dtype = xa_k_3_to_fp16_dtype_0, x = xa_k_3)[name = tensor("cast_60")]; tensor transpose_111 = transpose(perm = transpose_111_perm_0, x = xa_k_3_to_fp16)[name = tensor("transpose_228")]; tensor transpose_110 = transpose(perm = transpose_110_perm_0, x = xq_proj_7_cast_fp16)[name = tensor("transpose_229")]; tensor var_1484_cast_fp16 = matmul(transpose_x = var_1484_transpose_x_0, transpose_y = var_1484_transpose_y_0, x = transpose_110, y = transpose_111)[name = tensor("op_1484_cast_fp16")]; @@ -505,7 +513,7 @@ program(1.0) tensor xprobs_7_cast_fp16 = softmax(axis = var_1505, x = xscores_15_cast_fp16)[name = tensor("xprobs_7_cast_fp16")]; tensor var_1508_transpose_x_0 = const()[name = tensor("op_1508_transpose_x_0"), val = tensor(false)]; tensor var_1508_transpose_y_0 = const()[name = tensor("op_1508_transpose_y_0"), val = tensor(false)]; - tensor xa_v_3_to_fp16 = cast(dtype = xa_v_3_to_fp16_dtype_0, x = xa_v_3)[name = tensor("cast_33")]; + tensor xa_v_3_to_fp16 = cast(dtype = xa_v_3_to_fp16_dtype_0, x = xa_v_3)[name = tensor("cast_59")]; tensor xvT_7_cast_fp16 = transpose(perm = var_1482, x = xa_v_3_to_fp16)[name = tensor("transpose_227")]; tensor var_1508_cast_fp16 = matmul(transpose_x = var_1508_transpose_x_0, transpose_y = var_1508_transpose_y_0, x = xprobs_7_cast_fp16, y = xvT_7_cast_fp16)[name = tensor("op_1508_cast_fp16")]; tensor var_1513 = const()[name = tensor("op_1513"), val = tensor([0, 2, 1, 3])]; @@ -564,21 +572,23 @@ program(1.0) tensor new_v_9_squeeze_mask_0 = const()[name = tensor("new_v_9_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_9_cast_fp16 = slice_by_index(begin = new_v_9_begin_0, end = new_v_9_end_0, end_mask = new_v_9_end_mask_0, squeeze_mask = new_v_9_squeeze_mask_0, x = qkv_19_cast_fp16)[name = tensor("new_v_9_cast_fp16")]; tensor sa_k_in_4_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_4_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_4_to_fp16 = cast(dtype = sa_k_in_4_to_fp16_dtype_0, x = sa_k_in_4)[name = tensor("cast_32")]; + tensor sa_k_in_4_to_fp16 = cast(dtype = sa_k_in_4_to_fp16_dtype_0, x = sa_k_in_4)[name = tensor("cast_58")]; tensor var_1640_cast_fp16 = mul(x = sa_k_in_4_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1640_cast_fp16")]; tensor var_1641_cast_fp16 = mul(x = new_k_9_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1641_cast_fp16")]; - tensor sa_k_out_4 = add(x = var_1640_cast_fp16, y = var_1641_cast_fp16)[name = tensor("sa_k_out_9_cast_fp16")]; + tensor sa_k_out_9_cast_fp16 = add(x = var_1640_cast_fp16, y = var_1641_cast_fp16)[name = tensor("sa_k_out_9_cast_fp16")]; + tensor sa_k_out_9_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_9_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_4_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_4_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_4_to_fp16 = cast(dtype = sa_v_in_4_to_fp16_dtype_0, x = sa_v_in_4)[name = tensor("cast_31")]; + tensor sa_v_in_4_to_fp16 = cast(dtype = sa_v_in_4_to_fp16_dtype_0, x = sa_v_in_4)[name = tensor("cast_57")]; tensor var_1647_cast_fp16 = mul(x = sa_v_in_4_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1647_cast_fp16")]; tensor var_1648_cast_fp16 = mul(x = new_v_9_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1648_cast_fp16")]; - tensor sa_v_out_4 = add(x = var_1647_cast_fp16, y = var_1648_cast_fp16)[name = tensor("sa_v_out_9_cast_fp16")]; + tensor sa_v_out_9_cast_fp16 = add(x = var_1647_cast_fp16, y = var_1648_cast_fp16)[name = tensor("sa_v_out_9_cast_fp16")]; + tensor sa_v_out_9_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_9_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_1667 = const()[name = tensor("op_1667"), val = tensor([0, 2, -3, -1])]; tensor var_1669_transpose_x_0 = const()[name = tensor("op_1669_transpose_x_0"), val = tensor(false)]; tensor var_1669_transpose_y_0 = const()[name = tensor("op_1669_transpose_y_0"), val = tensor(false)]; tensor transpose_112_perm_0 = const()[name = tensor("transpose_112_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_113_perm_0 = const()[name = tensor("transpose_113_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_113 = transpose(perm = transpose_113_perm_0, x = sa_k_out_4)[name = tensor("transpose_222")]; + tensor transpose_113 = transpose(perm = transpose_113_perm_0, x = sa_k_out_9_cast_fp16)[name = tensor("transpose_222")]; tensor transpose_112 = transpose(perm = transpose_112_perm_0, x = q_9_cast_fp16)[name = tensor("transpose_223")]; tensor var_1669_cast_fp16 = matmul(transpose_x = var_1669_transpose_x_0, transpose_y = var_1669_transpose_y_0, x = transpose_112, y = transpose_113)[name = tensor("op_1669_cast_fp16")]; tensor var_1670_to_fp16 = const()[name = tensor("op_1670_to_fp16"), val = tensor(0x1p-3)]; @@ -589,7 +599,7 @@ program(1.0) tensor probs_9_cast_fp16 = softmax(axis = var_1690, x = scores_19_cast_fp16)[name = tensor("probs_9_cast_fp16")]; tensor var_1693_transpose_x_0 = const()[name = tensor("op_1693_transpose_x_0"), val = tensor(false)]; tensor var_1693_transpose_y_0 = const()[name = tensor("op_1693_transpose_y_0"), val = tensor(false)]; - tensor v_t_9_cast_fp16 = transpose(perm = var_1667, x = sa_v_out_4)[name = tensor("transpose_221")]; + tensor v_t_9_cast_fp16 = transpose(perm = var_1667, x = sa_v_out_9_cast_fp16)[name = tensor("transpose_221")]; tensor var_1693_cast_fp16 = matmul(transpose_x = var_1693_transpose_x_0, transpose_y = var_1693_transpose_y_0, x = probs_9_cast_fp16, y = v_t_9_cast_fp16)[name = tensor("op_1693_cast_fp16")]; tensor var_1698 = const()[name = tensor("op_1698"), val = tensor([0, 2, 1, 3])]; tensor var_1703 = const()[name = tensor("op_1703"), val = tensor([1, 1, -1])]; @@ -613,7 +623,7 @@ program(1.0) tensor xa_k_4_to_fp16_dtype_0 = const()[name = tensor("xa_k_4_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_114_perm_0 = const()[name = tensor("transpose_114_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_115_perm_0 = const()[name = tensor("transpose_115_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_4_to_fp16 = cast(dtype = xa_k_4_to_fp16_dtype_0, x = xa_k_4)[name = tensor("cast_30")]; + tensor xa_k_4_to_fp16 = cast(dtype = xa_k_4_to_fp16_dtype_0, x = xa_k_4)[name = tensor("cast_56")]; tensor transpose_115 = transpose(perm = transpose_115_perm_0, x = xa_k_4_to_fp16)[name = tensor("transpose_218")]; tensor transpose_114 = transpose(perm = transpose_114_perm_0, x = xq_proj_9_cast_fp16)[name = tensor("transpose_219")]; tensor var_1744_cast_fp16 = matmul(transpose_x = var_1744_transpose_x_0, transpose_y = var_1744_transpose_y_0, x = transpose_114, y = transpose_115)[name = tensor("op_1744_cast_fp16")]; @@ -625,7 +635,7 @@ program(1.0) tensor xprobs_9_cast_fp16 = softmax(axis = var_1765, x = xscores_19_cast_fp16)[name = tensor("xprobs_9_cast_fp16")]; tensor var_1768_transpose_x_0 = const()[name = tensor("op_1768_transpose_x_0"), val = tensor(false)]; tensor var_1768_transpose_y_0 = const()[name = tensor("op_1768_transpose_y_0"), val = tensor(false)]; - tensor xa_v_4_to_fp16 = cast(dtype = xa_v_4_to_fp16_dtype_0, x = xa_v_4)[name = tensor("cast_29")]; + tensor xa_v_4_to_fp16 = cast(dtype = xa_v_4_to_fp16_dtype_0, x = xa_v_4)[name = tensor("cast_55")]; tensor xvT_9_cast_fp16 = transpose(perm = var_1742, x = xa_v_4_to_fp16)[name = tensor("transpose_217")]; tensor var_1768_cast_fp16 = matmul(transpose_x = var_1768_transpose_x_0, transpose_y = var_1768_transpose_y_0, x = xprobs_9_cast_fp16, y = xvT_9_cast_fp16)[name = tensor("op_1768_cast_fp16")]; tensor var_1773 = const()[name = tensor("op_1773"), val = tensor([0, 2, 1, 3])]; @@ -684,21 +694,23 @@ program(1.0) tensor new_v_11_squeeze_mask_0 = const()[name = tensor("new_v_11_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_11_cast_fp16 = slice_by_index(begin = new_v_11_begin_0, end = new_v_11_end_0, end_mask = new_v_11_end_mask_0, squeeze_mask = new_v_11_squeeze_mask_0, x = qkv_23_cast_fp16)[name = tensor("new_v_11_cast_fp16")]; tensor sa_k_in_5_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_5_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_5_to_fp16 = cast(dtype = sa_k_in_5_to_fp16_dtype_0, x = sa_k_in_5)[name = tensor("cast_28")]; + tensor sa_k_in_5_to_fp16 = cast(dtype = sa_k_in_5_to_fp16_dtype_0, x = sa_k_in_5)[name = tensor("cast_54")]; tensor var_1900_cast_fp16 = mul(x = sa_k_in_5_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1900_cast_fp16")]; tensor var_1901_cast_fp16 = mul(x = new_k_11_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1901_cast_fp16")]; - tensor sa_k_out_5 = add(x = var_1900_cast_fp16, y = var_1901_cast_fp16)[name = tensor("sa_k_out_11_cast_fp16")]; + tensor sa_k_out_11_cast_fp16 = add(x = var_1900_cast_fp16, y = var_1901_cast_fp16)[name = tensor("sa_k_out_11_cast_fp16")]; + tensor sa_k_out_11_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_11_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_5_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_5_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_5_to_fp16 = cast(dtype = sa_v_in_5_to_fp16_dtype_0, x = sa_v_in_5)[name = tensor("cast_27")]; + tensor sa_v_in_5_to_fp16 = cast(dtype = sa_v_in_5_to_fp16_dtype_0, x = sa_v_in_5)[name = tensor("cast_53")]; tensor var_1907_cast_fp16 = mul(x = sa_v_in_5_to_fp16, y = var_599_cast_fp16)[name = tensor("op_1907_cast_fp16")]; tensor var_1908_cast_fp16 = mul(x = new_v_11_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_1908_cast_fp16")]; - tensor sa_v_out_5 = add(x = var_1907_cast_fp16, y = var_1908_cast_fp16)[name = tensor("sa_v_out_11_cast_fp16")]; + tensor sa_v_out_11_cast_fp16 = add(x = var_1907_cast_fp16, y = var_1908_cast_fp16)[name = tensor("sa_v_out_11_cast_fp16")]; + tensor sa_v_out_11_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_11_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_1927 = const()[name = tensor("op_1927"), val = tensor([0, 2, -3, -1])]; tensor var_1929_transpose_x_0 = const()[name = tensor("op_1929_transpose_x_0"), val = tensor(false)]; tensor var_1929_transpose_y_0 = const()[name = tensor("op_1929_transpose_y_0"), val = tensor(false)]; tensor transpose_116_perm_0 = const()[name = tensor("transpose_116_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_117_perm_0 = const()[name = tensor("transpose_117_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_117 = transpose(perm = transpose_117_perm_0, x = sa_k_out_5)[name = tensor("transpose_212")]; + tensor transpose_117 = transpose(perm = transpose_117_perm_0, x = sa_k_out_11_cast_fp16)[name = tensor("transpose_212")]; tensor transpose_116 = transpose(perm = transpose_116_perm_0, x = q_11_cast_fp16)[name = tensor("transpose_213")]; tensor var_1929_cast_fp16 = matmul(transpose_x = var_1929_transpose_x_0, transpose_y = var_1929_transpose_y_0, x = transpose_116, y = transpose_117)[name = tensor("op_1929_cast_fp16")]; tensor var_1930_to_fp16 = const()[name = tensor("op_1930_to_fp16"), val = tensor(0x1p-3)]; @@ -709,7 +721,7 @@ program(1.0) tensor probs_11_cast_fp16 = softmax(axis = var_1950, x = scores_23_cast_fp16)[name = tensor("probs_11_cast_fp16")]; tensor var_1953_transpose_x_0 = const()[name = tensor("op_1953_transpose_x_0"), val = tensor(false)]; tensor var_1953_transpose_y_0 = const()[name = tensor("op_1953_transpose_y_0"), val = tensor(false)]; - tensor v_t_11_cast_fp16 = transpose(perm = var_1927, x = sa_v_out_5)[name = tensor("transpose_211")]; + tensor v_t_11_cast_fp16 = transpose(perm = var_1927, x = sa_v_out_11_cast_fp16)[name = tensor("transpose_211")]; tensor var_1953_cast_fp16 = matmul(transpose_x = var_1953_transpose_x_0, transpose_y = var_1953_transpose_y_0, x = probs_11_cast_fp16, y = v_t_11_cast_fp16)[name = tensor("op_1953_cast_fp16")]; tensor var_1958 = const()[name = tensor("op_1958"), val = tensor([0, 2, 1, 3])]; tensor var_1963 = const()[name = tensor("op_1963"), val = tensor([1, 1, -1])]; @@ -733,7 +745,7 @@ program(1.0) tensor xa_k_5_to_fp16_dtype_0 = const()[name = tensor("xa_k_5_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_118_perm_0 = const()[name = tensor("transpose_118_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_119_perm_0 = const()[name = tensor("transpose_119_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_5_to_fp16 = cast(dtype = xa_k_5_to_fp16_dtype_0, x = xa_k_5)[name = tensor("cast_26")]; + tensor xa_k_5_to_fp16 = cast(dtype = xa_k_5_to_fp16_dtype_0, x = xa_k_5)[name = tensor("cast_52")]; tensor transpose_119 = transpose(perm = transpose_119_perm_0, x = xa_k_5_to_fp16)[name = tensor("transpose_208")]; tensor transpose_118 = transpose(perm = transpose_118_perm_0, x = xq_proj_11_cast_fp16)[name = tensor("transpose_209")]; tensor var_2004_cast_fp16 = matmul(transpose_x = var_2004_transpose_x_0, transpose_y = var_2004_transpose_y_0, x = transpose_118, y = transpose_119)[name = tensor("op_2004_cast_fp16")]; @@ -745,7 +757,7 @@ program(1.0) tensor xprobs_11_cast_fp16 = softmax(axis = var_2025, x = xscores_23_cast_fp16)[name = tensor("xprobs_11_cast_fp16")]; tensor var_2028_transpose_x_0 = const()[name = tensor("op_2028_transpose_x_0"), val = tensor(false)]; tensor var_2028_transpose_y_0 = const()[name = tensor("op_2028_transpose_y_0"), val = tensor(false)]; - tensor xa_v_5_to_fp16 = cast(dtype = xa_v_5_to_fp16_dtype_0, x = xa_v_5)[name = tensor("cast_25")]; + tensor xa_v_5_to_fp16 = cast(dtype = xa_v_5_to_fp16_dtype_0, x = xa_v_5)[name = tensor("cast_51")]; tensor xvT_11_cast_fp16 = transpose(perm = var_2002, x = xa_v_5_to_fp16)[name = tensor("transpose_207")]; tensor var_2028_cast_fp16 = matmul(transpose_x = var_2028_transpose_x_0, transpose_y = var_2028_transpose_y_0, x = xprobs_11_cast_fp16, y = xvT_11_cast_fp16)[name = tensor("op_2028_cast_fp16")]; tensor var_2033 = const()[name = tensor("op_2033"), val = tensor([0, 2, 1, 3])]; @@ -804,21 +816,23 @@ program(1.0) tensor new_v_13_squeeze_mask_0 = const()[name = tensor("new_v_13_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_13_cast_fp16 = slice_by_index(begin = new_v_13_begin_0, end = new_v_13_end_0, end_mask = new_v_13_end_mask_0, squeeze_mask = new_v_13_squeeze_mask_0, x = qkv_27_cast_fp16)[name = tensor("new_v_13_cast_fp16")]; tensor sa_k_in_6_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_6_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_6_to_fp16 = cast(dtype = sa_k_in_6_to_fp16_dtype_0, x = sa_k_in_6)[name = tensor("cast_24")]; + tensor sa_k_in_6_to_fp16 = cast(dtype = sa_k_in_6_to_fp16_dtype_0, x = sa_k_in_6)[name = tensor("cast_50")]; tensor var_2160_cast_fp16 = mul(x = sa_k_in_6_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2160_cast_fp16")]; tensor var_2161_cast_fp16 = mul(x = new_k_13_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2161_cast_fp16")]; - tensor sa_k_out_6 = add(x = var_2160_cast_fp16, y = var_2161_cast_fp16)[name = tensor("sa_k_out_13_cast_fp16")]; + tensor sa_k_out_13_cast_fp16 = add(x = var_2160_cast_fp16, y = var_2161_cast_fp16)[name = tensor("sa_k_out_13_cast_fp16")]; + tensor sa_k_out_13_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_13_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_6_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_6_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_6_to_fp16 = cast(dtype = sa_v_in_6_to_fp16_dtype_0, x = sa_v_in_6)[name = tensor("cast_23")]; + tensor sa_v_in_6_to_fp16 = cast(dtype = sa_v_in_6_to_fp16_dtype_0, x = sa_v_in_6)[name = tensor("cast_49")]; tensor var_2167_cast_fp16 = mul(x = sa_v_in_6_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2167_cast_fp16")]; tensor var_2168_cast_fp16 = mul(x = new_v_13_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2168_cast_fp16")]; - tensor sa_v_out_6 = add(x = var_2167_cast_fp16, y = var_2168_cast_fp16)[name = tensor("sa_v_out_13_cast_fp16")]; + tensor sa_v_out_13_cast_fp16 = add(x = var_2167_cast_fp16, y = var_2168_cast_fp16)[name = tensor("sa_v_out_13_cast_fp16")]; + tensor sa_v_out_13_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_13_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_2187 = const()[name = tensor("op_2187"), val = tensor([0, 2, -3, -1])]; tensor var_2189_transpose_x_0 = const()[name = tensor("op_2189_transpose_x_0"), val = tensor(false)]; tensor var_2189_transpose_y_0 = const()[name = tensor("op_2189_transpose_y_0"), val = tensor(false)]; tensor transpose_120_perm_0 = const()[name = tensor("transpose_120_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_121_perm_0 = const()[name = tensor("transpose_121_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_121 = transpose(perm = transpose_121_perm_0, x = sa_k_out_6)[name = tensor("transpose_202")]; + tensor transpose_121 = transpose(perm = transpose_121_perm_0, x = sa_k_out_13_cast_fp16)[name = tensor("transpose_202")]; tensor transpose_120 = transpose(perm = transpose_120_perm_0, x = q_13_cast_fp16)[name = tensor("transpose_203")]; tensor var_2189_cast_fp16 = matmul(transpose_x = var_2189_transpose_x_0, transpose_y = var_2189_transpose_y_0, x = transpose_120, y = transpose_121)[name = tensor("op_2189_cast_fp16")]; tensor var_2190_to_fp16 = const()[name = tensor("op_2190_to_fp16"), val = tensor(0x1p-3)]; @@ -829,7 +843,7 @@ program(1.0) tensor probs_13_cast_fp16 = softmax(axis = var_2210, x = scores_27_cast_fp16)[name = tensor("probs_13_cast_fp16")]; tensor var_2213_transpose_x_0 = const()[name = tensor("op_2213_transpose_x_0"), val = tensor(false)]; tensor var_2213_transpose_y_0 = const()[name = tensor("op_2213_transpose_y_0"), val = tensor(false)]; - tensor v_t_13_cast_fp16 = transpose(perm = var_2187, x = sa_v_out_6)[name = tensor("transpose_201")]; + tensor v_t_13_cast_fp16 = transpose(perm = var_2187, x = sa_v_out_13_cast_fp16)[name = tensor("transpose_201")]; tensor var_2213_cast_fp16 = matmul(transpose_x = var_2213_transpose_x_0, transpose_y = var_2213_transpose_y_0, x = probs_13_cast_fp16, y = v_t_13_cast_fp16)[name = tensor("op_2213_cast_fp16")]; tensor var_2218 = const()[name = tensor("op_2218"), val = tensor([0, 2, 1, 3])]; tensor var_2223 = const()[name = tensor("op_2223"), val = tensor([1, 1, -1])]; @@ -853,7 +867,7 @@ program(1.0) tensor xa_k_6_to_fp16_dtype_0 = const()[name = tensor("xa_k_6_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_122_perm_0 = const()[name = tensor("transpose_122_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_123_perm_0 = const()[name = tensor("transpose_123_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_6_to_fp16 = cast(dtype = xa_k_6_to_fp16_dtype_0, x = xa_k_6)[name = tensor("cast_22")]; + tensor xa_k_6_to_fp16 = cast(dtype = xa_k_6_to_fp16_dtype_0, x = xa_k_6)[name = tensor("cast_48")]; tensor transpose_123 = transpose(perm = transpose_123_perm_0, x = xa_k_6_to_fp16)[name = tensor("transpose_198")]; tensor transpose_122 = transpose(perm = transpose_122_perm_0, x = xq_proj_13_cast_fp16)[name = tensor("transpose_199")]; tensor var_2264_cast_fp16 = matmul(transpose_x = var_2264_transpose_x_0, transpose_y = var_2264_transpose_y_0, x = transpose_122, y = transpose_123)[name = tensor("op_2264_cast_fp16")]; @@ -865,7 +879,7 @@ program(1.0) tensor xprobs_13_cast_fp16 = softmax(axis = var_2285, x = xscores_27_cast_fp16)[name = tensor("xprobs_13_cast_fp16")]; tensor var_2288_transpose_x_0 = const()[name = tensor("op_2288_transpose_x_0"), val = tensor(false)]; tensor var_2288_transpose_y_0 = const()[name = tensor("op_2288_transpose_y_0"), val = tensor(false)]; - tensor xa_v_6_to_fp16 = cast(dtype = xa_v_6_to_fp16_dtype_0, x = xa_v_6)[name = tensor("cast_21")]; + tensor xa_v_6_to_fp16 = cast(dtype = xa_v_6_to_fp16_dtype_0, x = xa_v_6)[name = tensor("cast_47")]; tensor xvT_13_cast_fp16 = transpose(perm = var_2262, x = xa_v_6_to_fp16)[name = tensor("transpose_197")]; tensor var_2288_cast_fp16 = matmul(transpose_x = var_2288_transpose_x_0, transpose_y = var_2288_transpose_y_0, x = xprobs_13_cast_fp16, y = xvT_13_cast_fp16)[name = tensor("op_2288_cast_fp16")]; tensor var_2293 = const()[name = tensor("op_2293"), val = tensor([0, 2, 1, 3])]; @@ -924,21 +938,23 @@ program(1.0) tensor new_v_15_squeeze_mask_0 = const()[name = tensor("new_v_15_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_15_cast_fp16 = slice_by_index(begin = new_v_15_begin_0, end = new_v_15_end_0, end_mask = new_v_15_end_mask_0, squeeze_mask = new_v_15_squeeze_mask_0, x = qkv_31_cast_fp16)[name = tensor("new_v_15_cast_fp16")]; tensor sa_k_in_7_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_7_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_7_to_fp16 = cast(dtype = sa_k_in_7_to_fp16_dtype_0, x = sa_k_in_7)[name = tensor("cast_20")]; + tensor sa_k_in_7_to_fp16 = cast(dtype = sa_k_in_7_to_fp16_dtype_0, x = sa_k_in_7)[name = tensor("cast_46")]; tensor var_2420_cast_fp16 = mul(x = sa_k_in_7_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2420_cast_fp16")]; tensor var_2421_cast_fp16 = mul(x = new_k_15_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2421_cast_fp16")]; - tensor sa_k_out_7 = add(x = var_2420_cast_fp16, y = var_2421_cast_fp16)[name = tensor("sa_k_out_15_cast_fp16")]; + tensor sa_k_out_15_cast_fp16 = add(x = var_2420_cast_fp16, y = var_2421_cast_fp16)[name = tensor("sa_k_out_15_cast_fp16")]; + tensor sa_k_out_15_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_15_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_7_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_7_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_7_to_fp16 = cast(dtype = sa_v_in_7_to_fp16_dtype_0, x = sa_v_in_7)[name = tensor("cast_19")]; + tensor sa_v_in_7_to_fp16 = cast(dtype = sa_v_in_7_to_fp16_dtype_0, x = sa_v_in_7)[name = tensor("cast_45")]; tensor var_2427_cast_fp16 = mul(x = sa_v_in_7_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2427_cast_fp16")]; tensor var_2428_cast_fp16 = mul(x = new_v_15_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2428_cast_fp16")]; - tensor sa_v_out_7 = add(x = var_2427_cast_fp16, y = var_2428_cast_fp16)[name = tensor("sa_v_out_15_cast_fp16")]; + tensor sa_v_out_15_cast_fp16 = add(x = var_2427_cast_fp16, y = var_2428_cast_fp16)[name = tensor("sa_v_out_15_cast_fp16")]; + tensor sa_v_out_15_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_15_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_2447 = const()[name = tensor("op_2447"), val = tensor([0, 2, -3, -1])]; tensor var_2449_transpose_x_0 = const()[name = tensor("op_2449_transpose_x_0"), val = tensor(false)]; tensor var_2449_transpose_y_0 = const()[name = tensor("op_2449_transpose_y_0"), val = tensor(false)]; tensor transpose_124_perm_0 = const()[name = tensor("transpose_124_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_125_perm_0 = const()[name = tensor("transpose_125_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_125 = transpose(perm = transpose_125_perm_0, x = sa_k_out_7)[name = tensor("transpose_192")]; + tensor transpose_125 = transpose(perm = transpose_125_perm_0, x = sa_k_out_15_cast_fp16)[name = tensor("transpose_192")]; tensor transpose_124 = transpose(perm = transpose_124_perm_0, x = q_15_cast_fp16)[name = tensor("transpose_193")]; tensor var_2449_cast_fp16 = matmul(transpose_x = var_2449_transpose_x_0, transpose_y = var_2449_transpose_y_0, x = transpose_124, y = transpose_125)[name = tensor("op_2449_cast_fp16")]; tensor var_2450_to_fp16 = const()[name = tensor("op_2450_to_fp16"), val = tensor(0x1p-3)]; @@ -949,7 +965,7 @@ program(1.0) tensor probs_15_cast_fp16 = softmax(axis = var_2470, x = scores_31_cast_fp16)[name = tensor("probs_15_cast_fp16")]; tensor var_2473_transpose_x_0 = const()[name = tensor("op_2473_transpose_x_0"), val = tensor(false)]; tensor var_2473_transpose_y_0 = const()[name = tensor("op_2473_transpose_y_0"), val = tensor(false)]; - tensor v_t_15_cast_fp16 = transpose(perm = var_2447, x = sa_v_out_7)[name = tensor("transpose_191")]; + tensor v_t_15_cast_fp16 = transpose(perm = var_2447, x = sa_v_out_15_cast_fp16)[name = tensor("transpose_191")]; tensor var_2473_cast_fp16 = matmul(transpose_x = var_2473_transpose_x_0, transpose_y = var_2473_transpose_y_0, x = probs_15_cast_fp16, y = v_t_15_cast_fp16)[name = tensor("op_2473_cast_fp16")]; tensor var_2478 = const()[name = tensor("op_2478"), val = tensor([0, 2, 1, 3])]; tensor var_2483 = const()[name = tensor("op_2483"), val = tensor([1, 1, -1])]; @@ -973,7 +989,7 @@ program(1.0) tensor xa_k_7_to_fp16_dtype_0 = const()[name = tensor("xa_k_7_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_126_perm_0 = const()[name = tensor("transpose_126_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_127_perm_0 = const()[name = tensor("transpose_127_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_7_to_fp16 = cast(dtype = xa_k_7_to_fp16_dtype_0, x = xa_k_7)[name = tensor("cast_18")]; + tensor xa_k_7_to_fp16 = cast(dtype = xa_k_7_to_fp16_dtype_0, x = xa_k_7)[name = tensor("cast_44")]; tensor transpose_127 = transpose(perm = transpose_127_perm_0, x = xa_k_7_to_fp16)[name = tensor("transpose_188")]; tensor transpose_126 = transpose(perm = transpose_126_perm_0, x = xq_proj_15_cast_fp16)[name = tensor("transpose_189")]; tensor var_2524_cast_fp16 = matmul(transpose_x = var_2524_transpose_x_0, transpose_y = var_2524_transpose_y_0, x = transpose_126, y = transpose_127)[name = tensor("op_2524_cast_fp16")]; @@ -985,7 +1001,7 @@ program(1.0) tensor xprobs_15_cast_fp16 = softmax(axis = var_2545, x = xscores_31_cast_fp16)[name = tensor("xprobs_15_cast_fp16")]; tensor var_2548_transpose_x_0 = const()[name = tensor("op_2548_transpose_x_0"), val = tensor(false)]; tensor var_2548_transpose_y_0 = const()[name = tensor("op_2548_transpose_y_0"), val = tensor(false)]; - tensor xa_v_7_to_fp16 = cast(dtype = xa_v_7_to_fp16_dtype_0, x = xa_v_7)[name = tensor("cast_17")]; + tensor xa_v_7_to_fp16 = cast(dtype = xa_v_7_to_fp16_dtype_0, x = xa_v_7)[name = tensor("cast_43")]; tensor xvT_15_cast_fp16 = transpose(perm = var_2522, x = xa_v_7_to_fp16)[name = tensor("transpose_187")]; tensor var_2548_cast_fp16 = matmul(transpose_x = var_2548_transpose_x_0, transpose_y = var_2548_transpose_y_0, x = xprobs_15_cast_fp16, y = xvT_15_cast_fp16)[name = tensor("op_2548_cast_fp16")]; tensor var_2553 = const()[name = tensor("op_2553"), val = tensor([0, 2, 1, 3])]; @@ -1044,21 +1060,23 @@ program(1.0) tensor new_v_17_squeeze_mask_0 = const()[name = tensor("new_v_17_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_17_cast_fp16 = slice_by_index(begin = new_v_17_begin_0, end = new_v_17_end_0, end_mask = new_v_17_end_mask_0, squeeze_mask = new_v_17_squeeze_mask_0, x = qkv_35_cast_fp16)[name = tensor("new_v_17_cast_fp16")]; tensor sa_k_in_8_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_8_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_8_to_fp16 = cast(dtype = sa_k_in_8_to_fp16_dtype_0, x = sa_k_in_8)[name = tensor("cast_16")]; + tensor sa_k_in_8_to_fp16 = cast(dtype = sa_k_in_8_to_fp16_dtype_0, x = sa_k_in_8)[name = tensor("cast_42")]; tensor var_2680_cast_fp16 = mul(x = sa_k_in_8_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2680_cast_fp16")]; tensor var_2681_cast_fp16 = mul(x = new_k_17_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2681_cast_fp16")]; - tensor sa_k_out_8 = add(x = var_2680_cast_fp16, y = var_2681_cast_fp16)[name = tensor("sa_k_out_17_cast_fp16")]; + tensor sa_k_out_17_cast_fp16 = add(x = var_2680_cast_fp16, y = var_2681_cast_fp16)[name = tensor("sa_k_out_17_cast_fp16")]; + tensor sa_k_out_17_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_17_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_8_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_8_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_8_to_fp16 = cast(dtype = sa_v_in_8_to_fp16_dtype_0, x = sa_v_in_8)[name = tensor("cast_15")]; + tensor sa_v_in_8_to_fp16 = cast(dtype = sa_v_in_8_to_fp16_dtype_0, x = sa_v_in_8)[name = tensor("cast_41")]; tensor var_2687_cast_fp16 = mul(x = sa_v_in_8_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2687_cast_fp16")]; tensor var_2688_cast_fp16 = mul(x = new_v_17_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2688_cast_fp16")]; - tensor sa_v_out_8 = add(x = var_2687_cast_fp16, y = var_2688_cast_fp16)[name = tensor("sa_v_out_17_cast_fp16")]; + tensor sa_v_out_17_cast_fp16 = add(x = var_2687_cast_fp16, y = var_2688_cast_fp16)[name = tensor("sa_v_out_17_cast_fp16")]; + tensor sa_v_out_17_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_17_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_2707 = const()[name = tensor("op_2707"), val = tensor([0, 2, -3, -1])]; tensor var_2709_transpose_x_0 = const()[name = tensor("op_2709_transpose_x_0"), val = tensor(false)]; tensor var_2709_transpose_y_0 = const()[name = tensor("op_2709_transpose_y_0"), val = tensor(false)]; tensor transpose_128_perm_0 = const()[name = tensor("transpose_128_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_129_perm_0 = const()[name = tensor("transpose_129_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_129 = transpose(perm = transpose_129_perm_0, x = sa_k_out_8)[name = tensor("transpose_182")]; + tensor transpose_129 = transpose(perm = transpose_129_perm_0, x = sa_k_out_17_cast_fp16)[name = tensor("transpose_182")]; tensor transpose_128 = transpose(perm = transpose_128_perm_0, x = q_17_cast_fp16)[name = tensor("transpose_183")]; tensor var_2709_cast_fp16 = matmul(transpose_x = var_2709_transpose_x_0, transpose_y = var_2709_transpose_y_0, x = transpose_128, y = transpose_129)[name = tensor("op_2709_cast_fp16")]; tensor var_2710_to_fp16 = const()[name = tensor("op_2710_to_fp16"), val = tensor(0x1p-3)]; @@ -1069,7 +1087,7 @@ program(1.0) tensor probs_17_cast_fp16 = softmax(axis = var_2730, x = scores_35_cast_fp16)[name = tensor("probs_17_cast_fp16")]; tensor var_2733_transpose_x_0 = const()[name = tensor("op_2733_transpose_x_0"), val = tensor(false)]; tensor var_2733_transpose_y_0 = const()[name = tensor("op_2733_transpose_y_0"), val = tensor(false)]; - tensor v_t_17_cast_fp16 = transpose(perm = var_2707, x = sa_v_out_8)[name = tensor("transpose_181")]; + tensor v_t_17_cast_fp16 = transpose(perm = var_2707, x = sa_v_out_17_cast_fp16)[name = tensor("transpose_181")]; tensor var_2733_cast_fp16 = matmul(transpose_x = var_2733_transpose_x_0, transpose_y = var_2733_transpose_y_0, x = probs_17_cast_fp16, y = v_t_17_cast_fp16)[name = tensor("op_2733_cast_fp16")]; tensor var_2738 = const()[name = tensor("op_2738"), val = tensor([0, 2, 1, 3])]; tensor var_2743 = const()[name = tensor("op_2743"), val = tensor([1, 1, -1])]; @@ -1093,7 +1111,7 @@ program(1.0) tensor xa_k_8_to_fp16_dtype_0 = const()[name = tensor("xa_k_8_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_130_perm_0 = const()[name = tensor("transpose_130_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_131_perm_0 = const()[name = tensor("transpose_131_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_8_to_fp16 = cast(dtype = xa_k_8_to_fp16_dtype_0, x = xa_k_8)[name = tensor("cast_14")]; + tensor xa_k_8_to_fp16 = cast(dtype = xa_k_8_to_fp16_dtype_0, x = xa_k_8)[name = tensor("cast_40")]; tensor transpose_131 = transpose(perm = transpose_131_perm_0, x = xa_k_8_to_fp16)[name = tensor("transpose_178")]; tensor transpose_130 = transpose(perm = transpose_130_perm_0, x = xq_proj_17_cast_fp16)[name = tensor("transpose_179")]; tensor var_2784_cast_fp16 = matmul(transpose_x = var_2784_transpose_x_0, transpose_y = var_2784_transpose_y_0, x = transpose_130, y = transpose_131)[name = tensor("op_2784_cast_fp16")]; @@ -1105,7 +1123,7 @@ program(1.0) tensor xprobs_17_cast_fp16 = softmax(axis = var_2805, x = xscores_35_cast_fp16)[name = tensor("xprobs_17_cast_fp16")]; tensor var_2808_transpose_x_0 = const()[name = tensor("op_2808_transpose_x_0"), val = tensor(false)]; tensor var_2808_transpose_y_0 = const()[name = tensor("op_2808_transpose_y_0"), val = tensor(false)]; - tensor xa_v_8_to_fp16 = cast(dtype = xa_v_8_to_fp16_dtype_0, x = xa_v_8)[name = tensor("cast_13")]; + tensor xa_v_8_to_fp16 = cast(dtype = xa_v_8_to_fp16_dtype_0, x = xa_v_8)[name = tensor("cast_39")]; tensor xvT_17_cast_fp16 = transpose(perm = var_2782, x = xa_v_8_to_fp16)[name = tensor("transpose_177")]; tensor var_2808_cast_fp16 = matmul(transpose_x = var_2808_transpose_x_0, transpose_y = var_2808_transpose_y_0, x = xprobs_17_cast_fp16, y = xvT_17_cast_fp16)[name = tensor("op_2808_cast_fp16")]; tensor var_2813 = const()[name = tensor("op_2813"), val = tensor([0, 2, 1, 3])]; @@ -1164,21 +1182,23 @@ program(1.0) tensor new_v_19_squeeze_mask_0 = const()[name = tensor("new_v_19_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_19_cast_fp16 = slice_by_index(begin = new_v_19_begin_0, end = new_v_19_end_0, end_mask = new_v_19_end_mask_0, squeeze_mask = new_v_19_squeeze_mask_0, x = qkv_39_cast_fp16)[name = tensor("new_v_19_cast_fp16")]; tensor sa_k_in_9_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_9_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_9_to_fp16 = cast(dtype = sa_k_in_9_to_fp16_dtype_0, x = sa_k_in_9)[name = tensor("cast_12")]; + tensor sa_k_in_9_to_fp16 = cast(dtype = sa_k_in_9_to_fp16_dtype_0, x = sa_k_in_9)[name = tensor("cast_38")]; tensor var_2940_cast_fp16 = mul(x = sa_k_in_9_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2940_cast_fp16")]; tensor var_2941_cast_fp16 = mul(x = new_k_19_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2941_cast_fp16")]; - tensor sa_k_out_9 = add(x = var_2940_cast_fp16, y = var_2941_cast_fp16)[name = tensor("sa_k_out_19_cast_fp16")]; + tensor sa_k_out_19_cast_fp16 = add(x = var_2940_cast_fp16, y = var_2941_cast_fp16)[name = tensor("sa_k_out_19_cast_fp16")]; + tensor sa_k_out_19_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_19_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_9_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_9_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_9_to_fp16 = cast(dtype = sa_v_in_9_to_fp16_dtype_0, x = sa_v_in_9)[name = tensor("cast_11")]; + tensor sa_v_in_9_to_fp16 = cast(dtype = sa_v_in_9_to_fp16_dtype_0, x = sa_v_in_9)[name = tensor("cast_37")]; tensor var_2947_cast_fp16 = mul(x = sa_v_in_9_to_fp16, y = var_599_cast_fp16)[name = tensor("op_2947_cast_fp16")]; tensor var_2948_cast_fp16 = mul(x = new_v_19_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_2948_cast_fp16")]; - tensor sa_v_out_9 = add(x = var_2947_cast_fp16, y = var_2948_cast_fp16)[name = tensor("sa_v_out_19_cast_fp16")]; + tensor sa_v_out_19_cast_fp16 = add(x = var_2947_cast_fp16, y = var_2948_cast_fp16)[name = tensor("sa_v_out_19_cast_fp16")]; + tensor sa_v_out_19_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_19_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_2967 = const()[name = tensor("op_2967"), val = tensor([0, 2, -3, -1])]; tensor var_2969_transpose_x_0 = const()[name = tensor("op_2969_transpose_x_0"), val = tensor(false)]; tensor var_2969_transpose_y_0 = const()[name = tensor("op_2969_transpose_y_0"), val = tensor(false)]; tensor transpose_132_perm_0 = const()[name = tensor("transpose_132_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_133_perm_0 = const()[name = tensor("transpose_133_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_133 = transpose(perm = transpose_133_perm_0, x = sa_k_out_9)[name = tensor("transpose_172")]; + tensor transpose_133 = transpose(perm = transpose_133_perm_0, x = sa_k_out_19_cast_fp16)[name = tensor("transpose_172")]; tensor transpose_132 = transpose(perm = transpose_132_perm_0, x = q_19_cast_fp16)[name = tensor("transpose_173")]; tensor var_2969_cast_fp16 = matmul(transpose_x = var_2969_transpose_x_0, transpose_y = var_2969_transpose_y_0, x = transpose_132, y = transpose_133)[name = tensor("op_2969_cast_fp16")]; tensor var_2970_to_fp16 = const()[name = tensor("op_2970_to_fp16"), val = tensor(0x1p-3)]; @@ -1189,7 +1209,7 @@ program(1.0) tensor probs_19_cast_fp16 = softmax(axis = var_2990, x = scores_39_cast_fp16)[name = tensor("probs_19_cast_fp16")]; tensor var_2993_transpose_x_0 = const()[name = tensor("op_2993_transpose_x_0"), val = tensor(false)]; tensor var_2993_transpose_y_0 = const()[name = tensor("op_2993_transpose_y_0"), val = tensor(false)]; - tensor v_t_19_cast_fp16 = transpose(perm = var_2967, x = sa_v_out_9)[name = tensor("transpose_171")]; + tensor v_t_19_cast_fp16 = transpose(perm = var_2967, x = sa_v_out_19_cast_fp16)[name = tensor("transpose_171")]; tensor var_2993_cast_fp16 = matmul(transpose_x = var_2993_transpose_x_0, transpose_y = var_2993_transpose_y_0, x = probs_19_cast_fp16, y = v_t_19_cast_fp16)[name = tensor("op_2993_cast_fp16")]; tensor var_2998 = const()[name = tensor("op_2998"), val = tensor([0, 2, 1, 3])]; tensor var_3003 = const()[name = tensor("op_3003"), val = tensor([1, 1, -1])]; @@ -1213,7 +1233,7 @@ program(1.0) tensor xa_k_9_to_fp16_dtype_0 = const()[name = tensor("xa_k_9_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_134_perm_0 = const()[name = tensor("transpose_134_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_135_perm_0 = const()[name = tensor("transpose_135_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_9_to_fp16 = cast(dtype = xa_k_9_to_fp16_dtype_0, x = xa_k_9)[name = tensor("cast_10")]; + tensor xa_k_9_to_fp16 = cast(dtype = xa_k_9_to_fp16_dtype_0, x = xa_k_9)[name = tensor("cast_36")]; tensor transpose_135 = transpose(perm = transpose_135_perm_0, x = xa_k_9_to_fp16)[name = tensor("transpose_168")]; tensor transpose_134 = transpose(perm = transpose_134_perm_0, x = xq_proj_19_cast_fp16)[name = tensor("transpose_169")]; tensor var_3044_cast_fp16 = matmul(transpose_x = var_3044_transpose_x_0, transpose_y = var_3044_transpose_y_0, x = transpose_134, y = transpose_135)[name = tensor("op_3044_cast_fp16")]; @@ -1225,7 +1245,7 @@ program(1.0) tensor xprobs_19_cast_fp16 = softmax(axis = var_3065, x = xscores_39_cast_fp16)[name = tensor("xprobs_19_cast_fp16")]; tensor var_3068_transpose_x_0 = const()[name = tensor("op_3068_transpose_x_0"), val = tensor(false)]; tensor var_3068_transpose_y_0 = const()[name = tensor("op_3068_transpose_y_0"), val = tensor(false)]; - tensor xa_v_9_to_fp16 = cast(dtype = xa_v_9_to_fp16_dtype_0, x = xa_v_9)[name = tensor("cast_9")]; + tensor xa_v_9_to_fp16 = cast(dtype = xa_v_9_to_fp16_dtype_0, x = xa_v_9)[name = tensor("cast_35")]; tensor xvT_19_cast_fp16 = transpose(perm = var_3042, x = xa_v_9_to_fp16)[name = tensor("transpose_167")]; tensor var_3068_cast_fp16 = matmul(transpose_x = var_3068_transpose_x_0, transpose_y = var_3068_transpose_y_0, x = xprobs_19_cast_fp16, y = xvT_19_cast_fp16)[name = tensor("op_3068_cast_fp16")]; tensor var_3073 = const()[name = tensor("op_3073"), val = tensor([0, 2, 1, 3])]; @@ -1284,21 +1304,23 @@ program(1.0) tensor new_v_21_squeeze_mask_0 = const()[name = tensor("new_v_21_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_21_cast_fp16 = slice_by_index(begin = new_v_21_begin_0, end = new_v_21_end_0, end_mask = new_v_21_end_mask_0, squeeze_mask = new_v_21_squeeze_mask_0, x = qkv_43_cast_fp16)[name = tensor("new_v_21_cast_fp16")]; tensor sa_k_in_10_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_10_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_10_to_fp16 = cast(dtype = sa_k_in_10_to_fp16_dtype_0, x = sa_k_in_10)[name = tensor("cast_8")]; + tensor sa_k_in_10_to_fp16 = cast(dtype = sa_k_in_10_to_fp16_dtype_0, x = sa_k_in_10)[name = tensor("cast_34")]; tensor var_3200_cast_fp16 = mul(x = sa_k_in_10_to_fp16, y = var_599_cast_fp16)[name = tensor("op_3200_cast_fp16")]; tensor var_3201_cast_fp16 = mul(x = new_k_21_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_3201_cast_fp16")]; - tensor sa_k_out_10 = add(x = var_3200_cast_fp16, y = var_3201_cast_fp16)[name = tensor("sa_k_out_21_cast_fp16")]; + tensor sa_k_out_21_cast_fp16 = add(x = var_3200_cast_fp16, y = var_3201_cast_fp16)[name = tensor("sa_k_out_21_cast_fp16")]; + tensor sa_k_out_21_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_21_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_10_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_10_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_10_to_fp16 = cast(dtype = sa_v_in_10_to_fp16_dtype_0, x = sa_v_in_10)[name = tensor("cast_7")]; + tensor sa_v_in_10_to_fp16 = cast(dtype = sa_v_in_10_to_fp16_dtype_0, x = sa_v_in_10)[name = tensor("cast_33")]; tensor var_3207_cast_fp16 = mul(x = sa_v_in_10_to_fp16, y = var_599_cast_fp16)[name = tensor("op_3207_cast_fp16")]; tensor var_3208_cast_fp16 = mul(x = new_v_21_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_3208_cast_fp16")]; - tensor sa_v_out_10 = add(x = var_3207_cast_fp16, y = var_3208_cast_fp16)[name = tensor("sa_v_out_21_cast_fp16")]; + tensor sa_v_out_21_cast_fp16 = add(x = var_3207_cast_fp16, y = var_3208_cast_fp16)[name = tensor("sa_v_out_21_cast_fp16")]; + tensor sa_v_out_21_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_21_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_3227 = const()[name = tensor("op_3227"), val = tensor([0, 2, -3, -1])]; tensor var_3229_transpose_x_0 = const()[name = tensor("op_3229_transpose_x_0"), val = tensor(false)]; tensor var_3229_transpose_y_0 = const()[name = tensor("op_3229_transpose_y_0"), val = tensor(false)]; tensor transpose_136_perm_0 = const()[name = tensor("transpose_136_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_137_perm_0 = const()[name = tensor("transpose_137_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_137 = transpose(perm = transpose_137_perm_0, x = sa_k_out_10)[name = tensor("transpose_162")]; + tensor transpose_137 = transpose(perm = transpose_137_perm_0, x = sa_k_out_21_cast_fp16)[name = tensor("transpose_162")]; tensor transpose_136 = transpose(perm = transpose_136_perm_0, x = q_21_cast_fp16)[name = tensor("transpose_163")]; tensor var_3229_cast_fp16 = matmul(transpose_x = var_3229_transpose_x_0, transpose_y = var_3229_transpose_y_0, x = transpose_136, y = transpose_137)[name = tensor("op_3229_cast_fp16")]; tensor var_3230_to_fp16 = const()[name = tensor("op_3230_to_fp16"), val = tensor(0x1p-3)]; @@ -1309,7 +1331,7 @@ program(1.0) tensor probs_21_cast_fp16 = softmax(axis = var_3250, x = scores_43_cast_fp16)[name = tensor("probs_21_cast_fp16")]; tensor var_3253_transpose_x_0 = const()[name = tensor("op_3253_transpose_x_0"), val = tensor(false)]; tensor var_3253_transpose_y_0 = const()[name = tensor("op_3253_transpose_y_0"), val = tensor(false)]; - tensor v_t_21_cast_fp16 = transpose(perm = var_3227, x = sa_v_out_10)[name = tensor("transpose_161")]; + tensor v_t_21_cast_fp16 = transpose(perm = var_3227, x = sa_v_out_21_cast_fp16)[name = tensor("transpose_161")]; tensor var_3253_cast_fp16 = matmul(transpose_x = var_3253_transpose_x_0, transpose_y = var_3253_transpose_y_0, x = probs_21_cast_fp16, y = v_t_21_cast_fp16)[name = tensor("op_3253_cast_fp16")]; tensor var_3258 = const()[name = tensor("op_3258"), val = tensor([0, 2, 1, 3])]; tensor var_3263 = const()[name = tensor("op_3263"), val = tensor([1, 1, -1])]; @@ -1333,7 +1355,7 @@ program(1.0) tensor xa_k_10_to_fp16_dtype_0 = const()[name = tensor("xa_k_10_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_138_perm_0 = const()[name = tensor("transpose_138_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_139_perm_0 = const()[name = tensor("transpose_139_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_10_to_fp16 = cast(dtype = xa_k_10_to_fp16_dtype_0, x = xa_k_10)[name = tensor("cast_6")]; + tensor xa_k_10_to_fp16 = cast(dtype = xa_k_10_to_fp16_dtype_0, x = xa_k_10)[name = tensor("cast_32")]; tensor transpose_139 = transpose(perm = transpose_139_perm_0, x = xa_k_10_to_fp16)[name = tensor("transpose_158")]; tensor transpose_138 = transpose(perm = transpose_138_perm_0, x = xq_proj_21_cast_fp16)[name = tensor("transpose_159")]; tensor var_3304_cast_fp16 = matmul(transpose_x = var_3304_transpose_x_0, transpose_y = var_3304_transpose_y_0, x = transpose_138, y = transpose_139)[name = tensor("op_3304_cast_fp16")]; @@ -1345,7 +1367,7 @@ program(1.0) tensor xprobs_21_cast_fp16 = softmax(axis = var_3325, x = xscores_43_cast_fp16)[name = tensor("xprobs_21_cast_fp16")]; tensor var_3328_transpose_x_0 = const()[name = tensor("op_3328_transpose_x_0"), val = tensor(false)]; tensor var_3328_transpose_y_0 = const()[name = tensor("op_3328_transpose_y_0"), val = tensor(false)]; - tensor xa_v_10_to_fp16 = cast(dtype = xa_v_10_to_fp16_dtype_0, x = xa_v_10)[name = tensor("cast_5")]; + tensor xa_v_10_to_fp16 = cast(dtype = xa_v_10_to_fp16_dtype_0, x = xa_v_10)[name = tensor("cast_31")]; tensor xvT_21_cast_fp16 = transpose(perm = var_3302, x = xa_v_10_to_fp16)[name = tensor("transpose_157")]; tensor var_3328_cast_fp16 = matmul(transpose_x = var_3328_transpose_x_0, transpose_y = var_3328_transpose_y_0, x = xprobs_21_cast_fp16, y = xvT_21_cast_fp16)[name = tensor("op_3328_cast_fp16")]; tensor var_3333 = const()[name = tensor("op_3333"), val = tensor([0, 2, 1, 3])]; @@ -1404,21 +1426,23 @@ program(1.0) tensor new_v_squeeze_mask_0 = const()[name = tensor("new_v_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; tensor new_v_cast_fp16 = slice_by_index(begin = new_v_begin_0, end = new_v_end_0, end_mask = new_v_end_mask_0, squeeze_mask = new_v_squeeze_mask_0, x = qkv_cast_fp16)[name = tensor("new_v_cast_fp16")]; tensor sa_k_in_11_to_fp16_dtype_0 = const()[name = tensor("sa_k_in_11_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_k_in_11_to_fp16 = cast(dtype = sa_k_in_11_to_fp16_dtype_0, x = sa_k_in_11)[name = tensor("cast_4")]; + tensor sa_k_in_11_to_fp16 = cast(dtype = sa_k_in_11_to_fp16_dtype_0, x = sa_k_in_11)[name = tensor("cast_30")]; tensor var_3460_cast_fp16 = mul(x = sa_k_in_11_to_fp16, y = var_599_cast_fp16)[name = tensor("op_3460_cast_fp16")]; tensor var_3461_cast_fp16 = mul(x = new_k_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_3461_cast_fp16")]; - tensor sa_k_out_11 = add(x = var_3460_cast_fp16, y = var_3461_cast_fp16)[name = tensor("sa_k_out_cast_fp16")]; + tensor sa_k_out_cast_fp16 = add(x = var_3460_cast_fp16, y = var_3461_cast_fp16)[name = tensor("sa_k_out_cast_fp16")]; + tensor sa_k_out_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_k_out_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor sa_v_in_11_to_fp16_dtype_0 = const()[name = tensor("sa_v_in_11_to_fp16_dtype_0"), val = tensor("fp16")]; - tensor sa_v_in_11_to_fp16 = cast(dtype = sa_v_in_11_to_fp16_dtype_0, x = sa_v_in_11)[name = tensor("cast_3")]; + tensor sa_v_in_11_to_fp16 = cast(dtype = sa_v_in_11_to_fp16_dtype_0, x = sa_v_in_11)[name = tensor("cast_29")]; tensor var_3467_cast_fp16 = mul(x = sa_v_in_11_to_fp16, y = var_599_cast_fp16)[name = tensor("op_3467_cast_fp16")]; tensor var_3468_cast_fp16 = mul(x = new_v_cast_fp16, y = write_oh_b_1_cast_fp16)[name = tensor("op_3468_cast_fp16")]; - tensor sa_v_out_11 = add(x = var_3467_cast_fp16, y = var_3468_cast_fp16)[name = tensor("sa_v_out_cast_fp16")]; + tensor sa_v_out_cast_fp16 = add(x = var_3467_cast_fp16, y = var_3468_cast_fp16)[name = tensor("sa_v_out_cast_fp16")]; + tensor sa_v_out_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("sa_v_out_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor var_3487 = const()[name = tensor("op_3487"), val = tensor([0, 2, -3, -1])]; tensor var_3489_transpose_x_0 = const()[name = tensor("op_3489_transpose_x_0"), val = tensor(false)]; tensor var_3489_transpose_y_0 = const()[name = tensor("op_3489_transpose_y_0"), val = tensor(false)]; tensor transpose_140_perm_0 = const()[name = tensor("transpose_140_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_141_perm_0 = const()[name = tensor("transpose_141_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor transpose_141 = transpose(perm = transpose_141_perm_0, x = sa_k_out_11)[name = tensor("transpose_152")]; + tensor transpose_141 = transpose(perm = transpose_141_perm_0, x = sa_k_out_cast_fp16)[name = tensor("transpose_152")]; tensor transpose_140 = transpose(perm = transpose_140_perm_0, x = q_cast_fp16)[name = tensor("transpose_153")]; tensor var_3489_cast_fp16 = matmul(transpose_x = var_3489_transpose_x_0, transpose_y = var_3489_transpose_y_0, x = transpose_140, y = transpose_141)[name = tensor("op_3489_cast_fp16")]; tensor var_3490_to_fp16 = const()[name = tensor("op_3490_to_fp16"), val = tensor(0x1p-3)]; @@ -1429,7 +1453,7 @@ program(1.0) tensor probs_cast_fp16 = softmax(axis = var_3510, x = scores_cast_fp16)[name = tensor("probs_cast_fp16")]; tensor var_3513_transpose_x_0 = const()[name = tensor("op_3513_transpose_x_0"), val = tensor(false)]; tensor var_3513_transpose_y_0 = const()[name = tensor("op_3513_transpose_y_0"), val = tensor(false)]; - tensor v_t_cast_fp16 = transpose(perm = var_3487, x = sa_v_out_11)[name = tensor("transpose_151")]; + tensor v_t_cast_fp16 = transpose(perm = var_3487, x = sa_v_out_cast_fp16)[name = tensor("transpose_151")]; tensor var_3513_cast_fp16 = matmul(transpose_x = var_3513_transpose_x_0, transpose_y = var_3513_transpose_y_0, x = probs_cast_fp16, y = v_t_cast_fp16)[name = tensor("op_3513_cast_fp16")]; tensor var_3518 = const()[name = tensor("op_3518"), val = tensor([0, 2, 1, 3])]; tensor var_3523 = const()[name = tensor("op_3523"), val = tensor([1, 1, -1])]; @@ -1453,7 +1477,7 @@ program(1.0) tensor xa_k_11_to_fp16_dtype_0 = const()[name = tensor("xa_k_11_to_fp16_dtype_0"), val = tensor("fp16")]; tensor transpose_142_perm_0 = const()[name = tensor("transpose_142_perm_0"), val = tensor([0, 2, -3, -1])]; tensor transpose_143_perm_0 = const()[name = tensor("transpose_143_perm_0"), val = tensor([0, 2, -1, -3])]; - tensor xa_k_11_to_fp16 = cast(dtype = xa_k_11_to_fp16_dtype_0, x = xa_k_11)[name = tensor("cast_2")]; + tensor xa_k_11_to_fp16 = cast(dtype = xa_k_11_to_fp16_dtype_0, x = xa_k_11)[name = tensor("cast_28")]; tensor transpose_143 = transpose(perm = transpose_143_perm_0, x = xa_k_11_to_fp16)[name = tensor("transpose_148")]; tensor transpose_142 = transpose(perm = transpose_142_perm_0, x = xq_proj_cast_fp16)[name = tensor("transpose_149")]; tensor var_3564_cast_fp16 = matmul(transpose_x = var_3564_transpose_x_0, transpose_y = var_3564_transpose_y_0, x = transpose_142, y = transpose_143)[name = tensor("op_3564_cast_fp16")]; @@ -1465,7 +1489,7 @@ program(1.0) tensor xprobs_cast_fp16 = softmax(axis = var_3585, x = xscores_cast_fp16)[name = tensor("xprobs_cast_fp16")]; tensor var_3588_transpose_x_0 = const()[name = tensor("op_3588_transpose_x_0"), val = tensor(false)]; tensor var_3588_transpose_y_0 = const()[name = tensor("op_3588_transpose_y_0"), val = tensor(false)]; - tensor xa_v_11_to_fp16 = cast(dtype = xa_v_11_to_fp16_dtype_0, x = xa_v_11)[name = tensor("cast_1")]; + tensor xa_v_11_to_fp16 = cast(dtype = xa_v_11_to_fp16_dtype_0, x = xa_v_11)[name = tensor("cast_27")]; tensor xvT_cast_fp16 = transpose(perm = var_3562, x = xa_v_11_to_fp16)[name = tensor("transpose_147")]; tensor var_3588_cast_fp16 = matmul(transpose_x = var_3588_transpose_x_0, transpose_y = var_3588_transpose_y_0, x = xprobs_cast_fp16, y = xvT_cast_fp16)[name = tensor("op_3588_cast_fp16")]; tensor var_3593 = const()[name = tensor("op_3593"), val = tensor([0, 2, 1, 3])]; @@ -1503,12 +1527,40 @@ program(1.0) tensor input_axes_0 = const()[name = tensor("input_axes_0"), val = tensor([-1])]; tensor dec_norm_out_weight_to_fp16 = const()[name = tensor("dec_norm_out_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(89141824)))]; tensor var_3645_to_fp16 = const()[name = tensor("op_3645_to_fp16"), val = tensor(0x1.5p-17)]; - tensor h_last = layer_norm(axes = input_axes_0, epsilon = var_3645_to_fp16, gamma = dec_norm_out_weight_to_fp16, x = input_171_cast_fp16)[name = tensor("input_cast_fp16")]; + tensor input_cast_fp16 = layer_norm(axes = input_axes_0, epsilon = var_3645_to_fp16, gamma = dec_norm_out_weight_to_fp16, x = input_171_cast_fp16)[name = tensor("input_cast_fp16")]; + tensor input_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("input_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; tensor dec_final_proj_weight_to_fp16_quantized = constexpr_affine_dequantize()[axis = tensor(0), name = tensor("dec_final_proj_weight_to_fp16_quantized"), quantized_data = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(89143424))), scale = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(101595200))), zero_point = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(101578944)))]; tensor dec_final_proj_bias_to_fp16 = const()[name = tensor("dec_final_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(101627648)))]; - tensor linear_48_cast_fp16 = linear(bias = dec_final_proj_bias_to_fp16, weight = dec_final_proj_weight_to_fp16_quantized, x = h_last)[name = tensor("linear_48_cast_fp16")]; + tensor linear_48_cast_fp16 = linear(bias = dec_final_proj_bias_to_fp16, weight = dec_final_proj_weight_to_fp16_quantized, x = input_cast_fp16)[name = tensor("linear_48_cast_fp16")]; tensor var_3658 = const()[name = tensor("op_3658"), val = tensor([1, 1, 8, 2024])]; - tensor logits = reshape(shape = var_3658, x = linear_48_cast_fp16)[name = tensor("op_3659_cast_fp16")]; + tensor var_3659_cast_fp16 = reshape(shape = var_3658, x = linear_48_cast_fp16)[name = tensor("op_3659_cast_fp16")]; + tensor var_3659_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("op_3659_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor sa_k_out_0 = cast(dtype = sa_k_out_1_cast_fp16_to_fp32_dtype_0, x = sa_k_out_1_cast_fp16)[name = tensor("cast_1")]; + tensor sa_v_out_0 = cast(dtype = sa_v_out_1_cast_fp16_to_fp32_dtype_0, x = sa_v_out_1_cast_fp16)[name = tensor("cast_2")]; + tensor sa_k_out_1 = cast(dtype = sa_k_out_3_cast_fp16_to_fp32_dtype_0, x = sa_k_out_3_cast_fp16)[name = tensor("cast_3")]; + tensor sa_v_out_1 = cast(dtype = sa_v_out_3_cast_fp16_to_fp32_dtype_0, x = sa_v_out_3_cast_fp16)[name = tensor("cast_4")]; + tensor sa_k_out_2 = cast(dtype = sa_k_out_5_cast_fp16_to_fp32_dtype_0, x = sa_k_out_5_cast_fp16)[name = tensor("cast_5")]; + tensor sa_v_out_2 = cast(dtype = sa_v_out_5_cast_fp16_to_fp32_dtype_0, x = sa_v_out_5_cast_fp16)[name = tensor("cast_6")]; + tensor sa_k_out_3 = cast(dtype = sa_k_out_7_cast_fp16_to_fp32_dtype_0, x = sa_k_out_7_cast_fp16)[name = tensor("cast_7")]; + tensor sa_v_out_3 = cast(dtype = sa_v_out_7_cast_fp16_to_fp32_dtype_0, x = sa_v_out_7_cast_fp16)[name = tensor("cast_8")]; + tensor sa_k_out_4 = cast(dtype = sa_k_out_9_cast_fp16_to_fp32_dtype_0, x = sa_k_out_9_cast_fp16)[name = tensor("cast_9")]; + tensor sa_v_out_4 = cast(dtype = sa_v_out_9_cast_fp16_to_fp32_dtype_0, x = sa_v_out_9_cast_fp16)[name = tensor("cast_10")]; + tensor sa_k_out_5 = cast(dtype = sa_k_out_11_cast_fp16_to_fp32_dtype_0, x = sa_k_out_11_cast_fp16)[name = tensor("cast_11")]; + tensor sa_v_out_5 = cast(dtype = sa_v_out_11_cast_fp16_to_fp32_dtype_0, x = sa_v_out_11_cast_fp16)[name = tensor("cast_12")]; + tensor sa_k_out_6 = cast(dtype = sa_k_out_13_cast_fp16_to_fp32_dtype_0, x = sa_k_out_13_cast_fp16)[name = tensor("cast_13")]; + tensor sa_v_out_6 = cast(dtype = sa_v_out_13_cast_fp16_to_fp32_dtype_0, x = sa_v_out_13_cast_fp16)[name = tensor("cast_14")]; + tensor sa_k_out_7 = cast(dtype = sa_k_out_15_cast_fp16_to_fp32_dtype_0, x = sa_k_out_15_cast_fp16)[name = tensor("cast_15")]; + tensor sa_v_out_7 = cast(dtype = sa_v_out_15_cast_fp16_to_fp32_dtype_0, x = sa_v_out_15_cast_fp16)[name = tensor("cast_16")]; + tensor sa_k_out_8 = cast(dtype = sa_k_out_17_cast_fp16_to_fp32_dtype_0, x = sa_k_out_17_cast_fp16)[name = tensor("cast_17")]; + tensor sa_v_out_8 = cast(dtype = sa_v_out_17_cast_fp16_to_fp32_dtype_0, x = sa_v_out_17_cast_fp16)[name = tensor("cast_18")]; + tensor sa_k_out_9 = cast(dtype = sa_k_out_19_cast_fp16_to_fp32_dtype_0, x = sa_k_out_19_cast_fp16)[name = tensor("cast_19")]; + tensor sa_v_out_9 = cast(dtype = sa_v_out_19_cast_fp16_to_fp32_dtype_0, x = sa_v_out_19_cast_fp16)[name = tensor("cast_20")]; + tensor sa_k_out_10 = cast(dtype = sa_k_out_21_cast_fp16_to_fp32_dtype_0, x = sa_k_out_21_cast_fp16)[name = tensor("cast_21")]; + tensor sa_v_out_10 = cast(dtype = sa_v_out_21_cast_fp16_to_fp32_dtype_0, x = sa_v_out_21_cast_fp16)[name = tensor("cast_22")]; + tensor sa_k_out_11 = cast(dtype = sa_k_out_cast_fp16_to_fp32_dtype_0, x = sa_k_out_cast_fp16)[name = tensor("cast_23")]; + tensor sa_v_out_11 = cast(dtype = sa_v_out_cast_fp16_to_fp32_dtype_0, x = sa_v_out_cast_fp16)[name = tensor("cast_24")]; + tensor h_last = cast(dtype = input_cast_fp16_to_fp32_dtype_0, x = input_cast_fp16)[name = tensor("cast_25")]; + tensor logits = cast(dtype = var_3659_cast_fp16_to_fp32_dtype_0, x = var_3659_cast_fp16)[name = tensor("cast_26")]; tensor encoder_output_tmp = identity(x = encoder_output)[name = tensor("encoder_output_tmp")]; } -> (logits, h_last, sa_k_out_0, sa_k_out_1, sa_k_out_2, sa_k_out_3, sa_k_out_4, sa_k_out_5, sa_k_out_6, sa_k_out_7, sa_k_out_8, sa_k_out_9, sa_k_out_10, sa_k_out_11, sa_v_out_0, sa_v_out_1, sa_v_out_2, sa_v_out_3, sa_v_out_4, sa_v_out_5, sa_v_out_6, sa_v_out_7, sa_v_out_8, sa_v_out_9, sa_v_out_10, sa_v_out_11); } \ No newline at end of file