klemenk commited on
Commit
6597ca9
·
verified ·
1 Parent(s): d84f8d0

Calibrate centered mode for causal-stack delay

Browse files
Files changed (4) hide show
  1. README.md +16 -14
  2. config.json +2 -2
  3. configuration_wavcoch.py +6 -1
  4. modeling_wavcoch.py +1 -1
README.md CHANGED
@@ -42,8 +42,8 @@ wavcoch = AutoModel.from_pretrained(
42
  causal_codes = wavcoch.quantize(waveform_tensor)
43
  assert torch.equal(causal_codes, wavcoch.quantize(waveform_tensor, mode="causal"))
44
 
45
- # Centered mode preserves the number of frames while centering the 1001-sample
46
- # analysis window on the token endpoint. It uses 421 left and 500 right zeros.
47
  centered_codes = wavcoch.quantize(waveform_tensor, mode="centered")
48
 
49
  codes = causal_codes
@@ -62,11 +62,12 @@ audio = wavcoch.decode_audio(codes)
62
  This repo includes a bundled vocoder and supports `decode_audio(...)` for end-to-end waveform synthesis.
63
 
64
  `mode="causal"` is the default and uses the original 921 samples of left
65
- padding. `mode="centered"` redistributes the same total padding as 421 samples
66
- on the left and 500 samples on the right. This keeps frame counts unchanged
67
- while moving the analysis-window center forward by 500 samples (31.25 ms at
68
- 16 kHz). Centered encoding uses future waveform context and is therefore not
69
- streaming-causal.
 
70
 
71
  The mode argument is accepted by `forward(...)`, `quantize(...)`, and
72
  `wav2coch(...)`. Decoding methods operate on codes and do not take a mode.
@@ -86,16 +87,17 @@ Validation used float32 with TF32 disabled:
86
  divisible by the 80-sample hop.
87
  - Six synthetic beep configurations produced the same 200 frames for causal
88
  and centered modes from one second of audio.
89
- - Centered decoded-cochleagram power advanced by 6 frames (30 ms, the nearest
90
- 5 ms token-grid value to 31.25 ms), with shifted-curve Pearson correlations
91
- from 0.986 to 0.999.
 
92
  - AuriStream7BDeep-40Pred float32 head-1 and head-40 surprisals were bit-exact
93
  between the base model's codes and this model's causal codes.
94
 
95
- Centered mode changes the encoded token sequence and uses 31.25 ms of future
96
- waveform context. Models trained on causal WavCoch codes should therefore be
97
- evaluated explicitly before centered codes are used for likelihood estimates
98
- or generation.
99
 
100
  When called with `output_hidden_states=True`, WavCoch exposes a single hidden-state layer:
101
  the post-FSQ projected embedding sequence used for direct probing.
 
42
  causal_codes = wavcoch.quantize(waveform_tensor)
43
  assert torch.equal(causal_codes, wavcoch.quantize(waveform_tensor, mode="causal"))
44
 
45
+ # Centered mode preserves the number of frames and compensates the measured
46
+ # end-to-end causal-stack delay. It uses 341 left and 580 right zeros.
47
  centered_codes = wavcoch.quantize(waveform_tensor, mode="centered")
48
 
49
  codes = causal_codes
 
62
  This repo includes a bundled vocoder and supports `decode_audio(...)` for end-to-end waveform synthesis.
63
 
64
  `mode="causal"` is the default and uses the original 921 samples of left
65
+ padding. `mode="centered"` redistributes the same total padding as 341 samples
66
+ on the left and 580 samples on the right. This keeps frame counts unchanged
67
+ while advancing the front end by 580 samples (36.25 ms at 16 kHz). Of this,
68
+ 31.25 ms removes the original analysis-window offset and one additional 5 ms
69
+ frame compensates the measured causal encoder/decoder response delay. Centered
70
+ encoding uses future waveform context and is therefore not streaming-causal.
71
 
72
  The mode argument is accepted by `forward(...)`, `quantize(...)`, and
73
  `wav2coch(...)`. Decoding methods operate on codes and do not take a mode.
 
87
  divisible by the 80-sample hop.
88
  - Six synthetic beep configurations produced the same 200 frames for causal
89
  and centered modes from one second of audio.
90
+ - An end-to-end padding sweep found that geometric endpoint centering alone
91
+ left decoded impulse and abrupt-tone power peaks 5 ms late. Advancing one
92
+ additional frame aligned the decoded impulse and the median onset/cutoff
93
+ peaks to 0 ms on the 5 ms output grid.
94
  - AuriStream7BDeep-40Pred float32 head-1 and head-40 surprisals were bit-exact
95
  between the base model's codes and this model's causal codes.
96
 
97
+ Centered mode changes the encoded token sequence and uses approximately
98
+ 36.25 ms of future waveform context relative to token endpoints. Models trained
99
+ on causal WavCoch codes should therefore be evaluated explicitly before
100
+ centered codes are used for likelihood estimates or generation.
101
 
102
  When called with `output_hidden_states=True`, WavCoch exposes a single hidden-state layer:
103
  the post-FSQ projected embedding sequence used for direct probing.
config.json CHANGED
@@ -56,8 +56,8 @@
56
  "causal",
57
  "centered"
58
  ],
59
- "centered_left_padding": 421,
60
- "centered_right_padding": 500,
61
  "hop_length": 80,
62
  "causal_convs": true,
63
  "encoder_layers": 8,
 
56
  "causal",
57
  "centered"
58
  ],
59
+ "centered_left_padding": 341,
60
+ "centered_right_padding": 580,
61
  "hop_length": 80,
62
  "causal_convs": true,
63
  "encoder_layers": 8,
configuration_wavcoch.py CHANGED
@@ -48,8 +48,13 @@ class WavCochConfig(PretrainedConfig):
48
  self.window_size = int(window_size)
49
  self.window_padding = int(window_padding)
50
  half_window = (self.window_size - 1) // 2
 
 
 
51
  self.centered_right_padding = int(
52
- half_window if centered_right_padding is None else centered_right_padding
 
 
53
  )
54
  expected_left_padding = self.window_padding - self.centered_right_padding
55
  self.centered_left_padding = int(
 
48
  self.window_size = int(window_size)
49
  self.window_padding = int(window_padding)
50
  half_window = (self.window_size - 1) // 2
51
+ # Advance one additional hop beyond geometric endpoint centering to
52
+ # compensate the measured causal encoder/decoder response delay.
53
+ default_centered_right_padding = half_window + int(hop_length)
54
  self.centered_right_padding = int(
55
+ default_centered_right_padding
56
+ if centered_right_padding is None
57
+ else centered_right_padding
58
  )
59
  expected_left_padding = self.window_padding - self.centered_right_padding
60
  self.centered_left_padding = int(
modeling_wavcoch.py CHANGED
@@ -399,7 +399,7 @@ class WavCoch(PreTrainedModel):
399
  self.hop_length = int(config.hop_length)
400
  self.window_padding = int(getattr(config, "window_padding", self.N - self.hop_length))
401
  self.centered_right_padding = int(
402
- getattr(config, "centered_right_padding", (self.N - 1) // 2)
403
  )
404
  self.centered_left_padding = int(
405
  getattr(config, "centered_left_padding", self.window_padding - self.centered_right_padding)
 
399
  self.hop_length = int(config.hop_length)
400
  self.window_padding = int(getattr(config, "window_padding", self.N - self.hop_length))
401
  self.centered_right_padding = int(
402
+ getattr(config, "centered_right_padding", (self.N - 1) // 2 + self.hop_length)
403
  )
404
  self.centered_left_padding = int(
405
  getattr(config, "centered_left_padding", self.window_padding - self.centered_right_padding)