batch processing for more real time

Author: SevaSk

AudioRecorder.py

@@ -7,7 +7,7 @@
 DYNAMIC_ENERGY_THRESHOLD = False
 
 class BaseRecorder:
-    def __init__(self, source, source_name):
+    def __init__(self, source):
         self.recorder = sr.Recognizer()
         self.recorder.energy_threshold = ENERGY_THRESHOLD
         self.recorder.dynamic_energy_threshold = DYNAMIC_ENERGY_THRESHOLD
@@ -16,7 +16,6 @@ def __init__(self, source, source_name):
             raise ValueError("audio source can't be None")
 
         self.source = source
-        self.source_name = source_name
 
     def adjust_for_noise(self, device_name, msg):
         print(f"[INFO] Adjusting for ambient noise from {device_name}. " + msg)
@@ -27,13 +26,13 @@ def adjust_for_noise(self, device_name, msg):
     def record_into_queue(self, audio_queue):
         def record_callback(_, audio:sr.AudioData) -> None:
             data = audio.get_raw_data()
-            audio_queue.put((self.source_name, data, datetime.utcnow()))
+            audio_queue.put((data, datetime.utcnow()))
 
         self.recorder.listen_in_background(self.source, record_callback, phrase_time_limit=RECORD_TIMEOUT)
 
 class DefaultMicRecorder(BaseRecorder):
     def __init__(self):
-        super().__init__(source=sr.Microphone(sample_rate=16000), source_name="You")
+        super().__init__(source=sr.Microphone(sample_rate=16000))
         self.adjust_for_noise("Default Mic", "Please make some noise from the Default Mic...")
 
 class DefaultSpeakerRecorder(BaseRecorder):
@@ -55,5 +54,5 @@ def __init__(self):
                                sample_rate=int(default_speakers["defaultSampleRate"]),
                                chunk_size=pyaudio.get_sample_size(pyaudio.paInt16),
                                channels=default_speakers["maxInputChannels"])
-        super().__init__(source=source, source_name="Speaker")
+        super().__init__(source=source)
         self.adjust_for_noise("Default Speaker", "Please make or play some noise from the Default Speaker...")

AudioTranscriber.py

@@ -37,26 +37,68 @@ def __init__(self, mic_source, speaker_source, model):
             }
         }
 
-    def transcribe_audio_queue(self, audio_queue):
+    def transcribe_audio_queue(self, speaker_queue, mic_queue):
+        import queue
+        
         while True:
-            who_spoke, data, time_spoken = audio_queue.get()
-            self.update_last_sample_and_phrase_status(who_spoke, data, time_spoken)
-            source_info = self.audio_sources[who_spoke]
-
-            text = ''
-            try:
-                fd, path = tempfile.mkstemp(suffix=".wav")
-                os.close(fd)
-                source_info["process_data_func"](source_info["last_sample"], path)
-                text = self.audio_model.get_transcription(path)
-            except Exception as e:
-                print(e)
-            finally:
-                os.unlink(path)
-
-            if text != '' and text.lower() != 'you':
-                self.update_transcript(who_spoke, text, time_spoken)
+            pending_transcriptions = []
+            
+            mic_data = []
+            while True:
+                try:
+                    data, time_spoken = mic_queue.get_nowait()
+                    self.update_last_sample_and_phrase_status("You", data, time_spoken)
+                    mic_data.append((data, time_spoken))
+                except queue.Empty:
+                    break
+                    
+            speaker_data = []
+            while True:
+                try:
+                    data, time_spoken = speaker_queue.get_nowait()
+                    self.update_last_sample_and_phrase_status("Speaker", data, time_spoken)
+                    speaker_data.append((data, time_spoken))
+                except queue.Empty:
+                    break
+            
+            if mic_data:
+                source_info = self.audio_sources["You"]
+                try:
+                    fd, path = tempfile.mkstemp(suffix=".wav")
+                    os.close(fd)
+                    source_info["process_data_func"](source_info["last_sample"], path)
+                    text = self.audio_model.get_transcription(path)
+                    if text != '' and text.lower() != 'you':
+                        latest_time = max(time for _, time in mic_data)
+                        pending_transcriptions.append(("You", text, latest_time))
+                except Exception as e:
+                    print(f"Transcription error for You: {e}")
+                finally:
+                    os.unlink(path)
+            
+            if speaker_data:
+                source_info = self.audio_sources["Speaker"]
+                try:
+                    fd, path = tempfile.mkstemp(suffix=".wav")
+                    os.close(fd)
+                    source_info["process_data_func"](source_info["last_sample"], path)
+                    text = self.audio_model.get_transcription(path)
+                    if text != '' and text.lower() != 'you':
+                        latest_time = max(time for _, time in speaker_data)
+                        pending_transcriptions.append(("Speaker", text, latest_time))
+                except Exception as e:
+                    print(f"Transcription error for Speaker: {e}")
+                finally:
+                    os.unlink(path)
+            
+            if pending_transcriptions:
+                pending_transcriptions.sort(key=lambda x: x[2])
+                for who_spoke, text, time_spoken in pending_transcriptions:
+                    self.update_transcript(who_spoke, text, time_spoken)
+                
                 self.transcript_changed_event.set()
+            
+            threading.Event().wait(0.1)
 
     def update_last_sample_and_phrase_status(self, who_spoke, data, time_spoken):
         source_info = self.audio_sources[who_spoke]

main.py

@@ -17,12 +17,15 @@ def update_transcript_UI(transcriber, textbox):
     write_in_textbox(textbox, transcript_string)
     textbox.after(300, update_transcript_UI, transcriber, textbox)
 
-def clear_context(transcriber, audio_queue):
+def clear_context(transcriber, speaker_queue, mic_queue):
     transcriber.clear_transcript_data()
-    with audio_queue.mutex:
-        audio_queue.queue.clear()
 
-def create_ui_components(root, transcriber, audio_queue):
+    with speaker_queue.mutex:
+        speaker_queue.queue.clear()
+    with mic_queue.mutex:
+        mic_queue.queue.clear()
+
+def create_ui_components(root, transcriber, speaker_queue, mic_queue):
     ctk.set_appearance_mode("dark")
     ctk.set_default_color_theme("dark-blue")
     root.title("Ecoute")
@@ -49,7 +52,7 @@ def create_ui_components(root, transcriber, audio_queue):
     clear_button = ctk.CTkButton(
         main_frame, 
         text="Clear Transcript", 
-        command=lambda: clear_context(transcriber, audio_queue)
+        command=lambda: clear_context(transcriber, speaker_queue, mic_queue)
     )
     clear_button.grid(row=1, column=0, sticky="ew", padx=10, pady=(0, 10))
 
@@ -63,24 +66,25 @@ def main():
         return
 
     root = ctk.CTk()
-    audio_queue = queue.Queue()
+    speaker_queue = queue.Queue()
+    mic_queue = queue.Queue()
 
     user_audio_recorder = AudioRecorder.DefaultMicRecorder()
-    user_audio_recorder.record_into_queue(audio_queue)
+    user_audio_recorder.record_into_queue(mic_queue)
 
     time.sleep(2)
 
     speaker_audio_recorder = AudioRecorder.DefaultSpeakerRecorder()
-    speaker_audio_recorder.record_into_queue(audio_queue)
+    speaker_audio_recorder.record_into_queue(speaker_queue)
 
     model = TranscriberModels.get_model('--api' in sys.argv)
 
     transcriber = AudioTranscriber(user_audio_recorder.source, speaker_audio_recorder.source, model)
-    transcribe = threading.Thread(target=transcriber.transcribe_audio_queue, args=(audio_queue,))
+    transcribe = threading.Thread(target=transcriber.transcribe_audio_queue, args=(speaker_queue, mic_queue))
     transcribe.daemon = True
     transcribe.start()
 
-    transcript_textbox = create_ui_components(root, transcriber, audio_queue)
+    transcript_textbox = create_ui_components(root, transcriber, speaker_queue, mic_queue)
 
     print("READY")