add src_key_bias for whisper cross attn

wenet-e2e · Feb 7, 2025 · eb39741 · eb39741
1 parent 6533f8f
commit eb39741
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 0 deletions.
diff --git a/examples/aishell/whisper/conf/finetune_whisper_largev3.yaml b/examples/aishell/whisper/conf/finetune_whisper_largev3.yaml
@@ -25,6 +25,7 @@ decoder_conf:
   gradient_checkpointing: true
   input_layer: embed_learnable_pe
   key_bias: false
+  src_key_bias: false
   linear_units: 5120
   normalize_before: true
   num_blocks: 32

diff --git a/examples/aishell/whisper/conf/finetune_whisper_largev3_conv2d4.yaml b/examples/aishell/whisper/conf/finetune_whisper_largev3_conv2d4.yaml
@@ -25,6 +25,7 @@ decoder_conf:
   gradient_checkpointing: true
   input_layer: embed_learnable_pe
   key_bias: false
+  src_key_bias: false
   linear_units: 5120
   normalize_before: true
   num_blocks: 32

diff --git a/examples/aishell/whisper/conf/finetune_whisper_largev3_conv2d4_onlyattn.yaml b/examples/aishell/whisper/conf/finetune_whisper_largev3_conv2d4_onlyattn.yaml
@@ -25,6 +25,7 @@ decoder_conf:
   gradient_checkpointing: true
   input_layer: embed_learnable_pe
   key_bias: false
+  src_key_bias: false
   linear_units: 5120
   normalize_before: true
   num_blocks: 32

diff --git a/examples/aishell/whisper/conf/finetune_whisper_largev3_onlyattn.yaml b/examples/aishell/whisper/conf/finetune_whisper_largev3_onlyattn.yaml
@@ -25,6 +25,7 @@ decoder_conf:
   gradient_checkpointing: true
   input_layer: embed_learnable_pe
   key_bias: false
+  src_key_bias: false
   linear_units: 5120
   normalize_before: true
   num_blocks: 32

diff --git a/examples/wenetspeech/whisper/conf/finetune_whisper_largev3.yaml b/examples/wenetspeech/whisper/conf/finetune_whisper_largev3.yaml
@@ -25,6 +25,7 @@ decoder_conf:
   gradient_checkpointing: true
   input_layer: embed_learnable_pe
   key_bias: false
+  src_key_bias: false
   linear_units: 5120
   normalize_before: true
   num_blocks: 32

diff --git a/wenet/whisper/convert_whisper_to_wenet_config_and_ckpt.py b/wenet/whisper/convert_whisper_to_wenet_config_and_ckpt.py
@@ -84,6 +84,7 @@ def convert_to_wenet_yaml(tokenizer, dims, wenet_yaml_path: str):
     configs['decoder_conf']['normalize_before'] = True
     configs['decoder_conf']['src_attention'] = True
     configs['decoder_conf']['key_bias'] = False
+    configs['decoder_conf']['src_key_bias'] = False
     configs['decoder_conf']['activation_type'] = "gelu"
 
     configs['tokenizer'] = 'whisper'