From d87f6268d07d112372bc6f8de37c79bf30bc63f8 Mon Sep 17 00:00:00 2001 From: briguetjo Date: Sat, 29 Jul 2023 18:13:36 +0200 Subject: [PATCH 1/4] fix preset language --- whisperx/asr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/whisperx/asr.py b/whisperx/asr.py index 09454c9..67b5b4c 100644 --- a/whisperx/asr.py +++ b/whisperx/asr.py @@ -1,6 +1,6 @@ import os import warnings -from typing import List, Union +from typing import List, Union, Optional import ctranslate2 import faster_whisper @@ -27,7 +27,7 @@ def load_model(whisper_arch, device_index=0, compute_type="float16", asr_options=None, - language=None, + language : Optional[str] = None, vad_options=None, model=None, task="transcribe", @@ -104,9 +104,7 @@ def load_model(whisper_arch, vad_model = load_vad_model(torch.device(device), use_auth_token=None, **default_vad_options) - return FasterWhisperPipeline(model, vad_model, default_asr_options, tokenizer) - - + return FasterWhisperPipeline(model, vad_model, default_asr_options, tokenizer, language) class WhisperModel(faster_whisper.WhisperModel): ''' @@ -183,11 +181,13 @@ class FasterWhisperPipeline(Pipeline): vad, options, tokenizer=None, + language : Optional[str] = None, device: Union[int, str, "torch.device"] = -1, framework = "pt", **kwargs ): self.model = model + self.preset_language = language self.tokenizer = tokenizer self.options = options self._batch_size = kwargs.pop("batch_size", None) @@ -258,7 +258,7 @@ class FasterWhisperPipeline(Pipeline): vad_segments = self.vad_model({"waveform": torch.from_numpy(audio).unsqueeze(0), "sample_rate": SAMPLE_RATE}) vad_segments = merge_chunks(vad_segments, 30) - if self.tokenizer is None: + if self.preset_language is None: language = language or self.detect_language(audio) task = task or "transcribe" self.tokenizer = faster_whisper.tokenizer.Tokenizer(self.model.hf_tokenizer, From 9d736dca1c79f856f8d96628bddec64d2a69ff7d Mon Sep 17 00:00:00 2001 From: briguetjo Date: Sat, 29 Jul 2023 18:20:59 +0200 Subject: [PATCH 2/4] add some warning if languages do not match --- whisperx/asr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/whisperx/asr.py b/whisperx/asr.py index 67b5b4c..623cf72 100644 --- a/whisperx/asr.py +++ b/whisperx/asr.py @@ -258,7 +258,10 @@ class FasterWhisperPipeline(Pipeline): vad_segments = self.vad_model({"waveform": torch.from_numpy(audio).unsqueeze(0), "sample_rate": SAMPLE_RATE}) vad_segments = merge_chunks(vad_segments, 30) - if self.preset_language is None: + if self.preset_language is None or self.preset_language != language: + if self.preset_language is not None and language is not None and self.preset_language != language: + print(f"Preset language '{self.preset_language}' is different from the language {language} passed to the transcribe method.") + print(f"Overriding preset language with {language}.") language = language or self.detect_language(audio) task = task or "transcribe" self.tokenizer = faster_whisper.tokenizer.Tokenizer(self.model.hf_tokenizer, From 864976af2385f8202484c4b56506c971e5395b31 Mon Sep 17 00:00:00 2001 From: briguetjo Date: Sat, 29 Jul 2023 18:56:33 +0200 Subject: [PATCH 3/4] fix issue by resetting tokenizer --- whisperx/asr.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/whisperx/asr.py b/whisperx/asr.py index 623cf72..cd3cea3 100644 --- a/whisperx/asr.py +++ b/whisperx/asr.py @@ -258,10 +258,7 @@ class FasterWhisperPipeline(Pipeline): vad_segments = self.vad_model({"waveform": torch.from_numpy(audio).unsqueeze(0), "sample_rate": SAMPLE_RATE}) vad_segments = merge_chunks(vad_segments, 30) - if self.preset_language is None or self.preset_language != language: - if self.preset_language is not None and language is not None and self.preset_language != language: - print(f"Preset language '{self.preset_language}' is different from the language {language} passed to the transcribe method.") - print(f"Overriding preset language with {language}.") + if self.tokenizer is None: language = language or self.detect_language(audio) task = task or "transcribe" self.tokenizer = faster_whisper.tokenizer.Tokenizer(self.model.hf_tokenizer, @@ -289,6 +286,9 @@ class FasterWhisperPipeline(Pipeline): } ) + if self.preset_language is None: + self.tokenizer = None + return {"segments": segments, "language": language} From 225f6b4d6915fd97a0066c116299559cdccf8732 Mon Sep 17 00:00:00 2001 From: briguetjo Date: Sat, 29 Jul 2023 19:34:51 +0200 Subject: [PATCH 4/4] fix suppress_numerals --- whisperx/asr.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/whisperx/asr.py b/whisperx/asr.py index cd3cea3..ecc2765 100644 --- a/whisperx/asr.py +++ b/whisperx/asr.py @@ -1,6 +1,6 @@ import os import warnings -from typing import List, Union, Optional +from typing import List, Union, Optional, NamedTuple import ctranslate2 import faster_whisper @@ -83,13 +83,7 @@ def load_model(whisper_arch, if asr_options is not None: default_asr_options.update(asr_options) - if default_asr_options["suppress_numerals"]: - if tokenizer is None: - tokenizer = faster_whisper.tokenizer.Tokenizer(model.hf_tokenizer, model.model.is_multilingual, task=task, language="en") - numeral_symbol_tokens = find_numeral_symbol_tokens(tokenizer) - print(f"Suppressing numeral and symbol tokens: {numeral_symbol_tokens}") - default_asr_options["suppress_tokens"] += numeral_symbol_tokens - default_asr_options["suppress_tokens"] = list(set(default_asr_options["suppress_tokens"])) + suppress_numerals = default_asr_options["suppress_numerals"] del default_asr_options["suppress_numerals"] default_asr_options = faster_whisper.transcribe.TranscriptionOptions(**default_asr_options) @@ -104,7 +98,14 @@ def load_model(whisper_arch, vad_model = load_vad_model(torch.device(device), use_auth_token=None, **default_vad_options) - return FasterWhisperPipeline(model, vad_model, default_asr_options, tokenizer, language) + return FasterWhisperPipeline( + model=model, + vad=vad_model, + options=default_asr_options, + tokenizer=tokenizer, + language=language, + suppress_numerals=suppress_numerals, + ) class WhisperModel(faster_whisper.WhisperModel): ''' @@ -179,17 +180,19 @@ class FasterWhisperPipeline(Pipeline): self, model, vad, - options, + options : NamedTuple, tokenizer=None, - language : Optional[str] = None, device: Union[int, str, "torch.device"] = -1, framework = "pt", + language : Optional[str] = None, + suppress_numerals: bool = False, **kwargs ): self.model = model - self.preset_language = language self.tokenizer = tokenizer self.options = options + self.preset_language = language + self.suppress_numerals = suppress_numerals self._batch_size = kwargs.pop("batch_size", None) self._num_workers = 1 self._preprocess_params, self._forward_params, self._postprocess_params = self._sanitize_parameters(**kwargs) @@ -271,6 +274,14 @@ class FasterWhisperPipeline(Pipeline): self.tokenizer = faster_whisper.tokenizer.Tokenizer(self.model.hf_tokenizer, self.model.model.is_multilingual, task=task, language=language) + + if self.suppress_numerals: + previous_suppress_tokens = self.options.suppress_tokens + numeral_symbol_tokens = find_numeral_symbol_tokens(self.tokenizer) + print(f"Suppressing numeral and symbol tokens: {numeral_symbol_tokens}") + new_suppressed_tokens = numeral_symbol_tokens + self.options.suppress_tokens + new_suppressed_tokens = list(set(new_suppressed_tokens)) + self.options = self.options._replace(suppress_tokens=new_suppressed_tokens) segments: List[SingleSegment] = [] batch_size = batch_size or self._batch_size @@ -286,9 +297,14 @@ class FasterWhisperPipeline(Pipeline): } ) + # revert the tokenizer if multilingual inference is enabled if self.preset_language is None: self.tokenizer = None + # revert suppressed tokens if suppress_numerals is enabled + if self.suppress_numerals: + self.options = self.options._replace(suppress_tokens=previous_suppress_tokens) + return {"segments": segments, "language": language}