default_stage:
  default_modifiers:
    AWQModifier:
      requires_calibration_data: true
      mappings:
      - smooth_layer: re:.*post_attention_layernorm$
        balance_layers: ['re:.*mlp\.gate_proj$', 're:.*mlp\.up_proj$']
        activation_hook_target: null
      - smooth_layer: re:.*mlp\.up_proj$
        balance_layers: ['re:.*mlp\.down_proj$']
        activation_hook_target: null
      offload_device: !!python/object/apply:torch.device [cpu]
      duo_scaling: true
      n_grid: 20
    GPTQModifier:
      config_groups:
        g0:
          targets: ['re:.*layers\.([0-9]|[1-4][0-9]|5[0-5])\.mlp\.(gate|up|down)_proj$']
          weights:
            num_bits: 4
            type: float
            symmetric: true
            group_size: 16
            strategy: tensor_group
            block_structure: null
            dynamic: false
            actorder: static
            scale_dtype: torch.float8_e4m3fn
            zp_dtype: null
            observer: imatrix_mse
            observer_kwargs: {}
          input_activations:
            num_bits: 4
            type: float
            symmetric: true
            group_size: 16
            strategy: tensor_group
            block_structure: null
            dynamic: local
            actorder: null
            scale_dtype: torch.float8_e4m3fn
            zp_dtype: null
            observer: static_minmax
            observer_kwargs: {}
          output_activations: null
          format: null
        g1:
          targets: ['re:.*self_attn\.(q|k|v|o)_proj$', 're:.*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$',
            're:.*layers\.(5[6-9]|6[0-3])\.mlp\.(gate|up|down)_proj$']
          weights:
            num_bits: 8
            type: float
            symmetric: true
            group_size: null
            strategy: channel
            block_structure: null
            dynamic: false
            actorder: static
            scale_dtype: null
            zp_dtype: null
            observer: memoryless_minmax
            observer_kwargs: {}
          input_activations:
            num_bits: 8
            type: float
            symmetric: true
            group_size: null
            strategy: token
            block_structure: null
            dynamic: true
            actorder: null
            scale_dtype: null
            zp_dtype: null
            observer: null
            observer_kwargs: {}
          output_activations: null
          format: null
      targets: ['re:.*layers\.([0-9]|[1-4][0-9]|5[0-5])\.mlp\.(gate|up|down)_proj$', 're:.*self_attn\.(q|k|v|o)_proj$',
        're:.*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$', 're:.*layers\.(5[6-9]|6[0-3])\.mlp\.(gate|up|down)_proj$']
      ignore: [lm_head, 're:visual.*', 're:model\.visual.*', 're:.*embed_tokens$', 're:.*mtp.*',
        're:.*linear_attn\.(in_proj_b|in_proj_a|conv1d|A_log|dt_bias|norm)$', 're:.*norm$']
      bypass_divisibility_checks: false
      requires_calibration_data: true
      block_size: 128
      dampening_frac: 0.01
      actorder: static
      offload_hessians: false