From ec53362d19972aab84cf1c5466fb289465bd2b3e Mon Sep 17 00:00:00 2001 From: Hanjin Liu Date: Thu, 9 Mar 2023 19:55:37 +0900 Subject: [PATCH 1/5] wip --- magicclass/ext/pyaudio/__init__.py | 0 magicclass/ext/pyaudio/widget.py | 3 +++ 2 files changed, 3 insertions(+) create mode 100644 magicclass/ext/pyaudio/__init__.py create mode 100644 magicclass/ext/pyaudio/widget.py diff --git a/magicclass/ext/pyaudio/__init__.py b/magicclass/ext/pyaudio/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/magicclass/ext/pyaudio/widget.py b/magicclass/ext/pyaudio/widget.py new file mode 100644 index 00000000..170e3977 --- /dev/null +++ b/magicclass/ext/pyaudio/widget.py @@ -0,0 +1,3 @@ +from __future__ import annotations +import numpy as np +from pyaudio import PyAudio From cfd9982e504afec2582ec807c72e4bc85da3040c Mon Sep 17 00:00:00 2001 From: Hanjin Liu Date: Sun, 12 Mar 2023 17:10:44 +0900 Subject: [PATCH 2/5] worked --- magicclass/ext/pyaudio/__init__.py | 3 + magicclass/ext/pyaudio/widget.py | 170 ++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 1 deletion(-) diff --git a/magicclass/ext/pyaudio/__init__.py b/magicclass/ext/pyaudio/__init__.py index e69de29b..0a585f9f 100644 --- a/magicclass/ext/pyaudio/__init__.py +++ b/magicclass/ext/pyaudio/__init__.py @@ -0,0 +1,3 @@ +from .widget import AudioRecorder + +__all__ = ["AudioRecorder"] diff --git a/magicclass/ext/pyaudio/widget.py b/magicclass/ext/pyaudio/widget.py index 170e3977..6061f8bf 100644 --- a/magicclass/ext/pyaudio/widget.py +++ b/magicclass/ext/pyaudio/widget.py @@ -1,3 +1,171 @@ from __future__ import annotations + +from qtpy import QtWidgets as QtW, QtCore, QtGui +from qtpy.QtCore import Qt, Signal import numpy as np -from pyaudio import PyAudio +from numpy.typing import NDArray +import pyaudio +from magicgui.backends._qtpy.widgets import QBaseValueWidget +from magicgui.application import use_app +from magicclass._magicgui_compat import _mpl_image, ValueWidget +from magicclass.widgets.utils import merge_super_sigs + + +class QAudioImage(QtW.QLabel): + resized = Signal() + + def __init__(self, parent: QtW.QWidget | None = None): + super().__init__(parent) + self._audio_data = np.zeros(0, dtype=np.int16) + self.resized.connect(self.update) + self._chunksize = 441 + + def update(self) -> None: + image = self._image_array() + + img = _mpl_image.Image() + + img.set_data(image) + + val = img.make_image() + h, w, _ = val.shape + qimage = QtGui.QImage(val, w, h, QtGui.QImage.Format.Format_RGBA8888) + _pixmap = QtGui.QPixmap.fromImage(qimage).scaled( + self.size(), + Qt.AspectRatioMode.IgnoreAspectRatio, + Qt.TransformationMode.SmoothTransformation, + ) + self.setPixmap(_pixmap) + self.setMinimumSize(108, 28) + super().update() + + def _image_array(self) -> NDArray[np.uint8]: + w, h = self.width(), self.height() + audio = _max_binning(self._audio_data, self._chunksize, w * 2) + half_array = np.stack( + [np.linspace(0, 2**16 - 1, h // 2)] * audio.size, axis=1 + ) + array = np.concatenate([half_array[::-1], half_array], axis=0) + binary = array < audio + image = np.full(array.shape + (4,), 248, dtype=np.uint8) + image[binary] = np.array([0, 0, 255, 255], dtype=np.uint8)[np.newaxis] + return image + + +def _max_binning(arr: NDArray[np.int16], n: int, width: int) -> NDArray[np.int16]: + nchunks, res = divmod(arr.size, n) + if nchunks < width: + out_left = np.abs(arr[:-res]).reshape(-1, n).max(axis=1) + out = np.concatenate([out_left, np.zeros(width - nchunks, dtype=np.int16)]) + else: + out = np.abs(arr[:-res]).reshape(-1, n)[-width:].max(axis=1) + return out + + +class QAudioRecorder(QtW.QWidget): + valueChanged = Signal(object) + + def __init__(self, parent: QtW.QWidget | None = None) -> None: + super().__init__(parent) + self._chunk = 1024 + self._rate = 44100 + self._format = pyaudio.paInt16 + self._audio = pyaudio.PyAudio() + self._stream = self._audio.open( + format=self._format, + channels=1, + rate=self._rate, + input=True, + frames_per_buffer=self._chunk, + ) + + self._timer = QtCore.QTimer() + self._timer.timeout.connect(self._update_data) + + self._setup_ui() + self._recoding = False + + def _setup_ui(self): + self._btn = QtW.QPushButton("Rec") + self._btn.setFixedSize(36, 24) + self._label = QAudioImage() + layout = QtW.QHBoxLayout() + layout.setContentsMargins(0, 0, 0, 0) + self.setLayout(layout) + layout.addWidget(self._label) + layout.addWidget(self._btn) + + self._btn.clicked.connect(self._on_button_clicked) + + def _update_data(self): + _incoming = self._read_input() + self.setValue(np.concatenate([self._label._audio_data, _incoming])) + + def _read_input(self): + ret_bytes = self._stream.read(self._chunk) + ret = np.frombuffer(ret_bytes, dtype=np.int16) + return ret + + def _on_button_clicked(self): + self.setRecording(not self.recording()) + + def recording(self) -> bool: + return self._recoding + + def setRecording(self, value: bool) -> None: + was_recording = self._recoding + self._recoding = value + if self._recoding and not was_recording: + self._timer.start(10) + self._btn.setText("Stop") + elif not self._recoding and was_recording: + self._timer.stop() + self._btn.setText("Rec") + + def value(self) -> NDArray[np.int16]: + return self._label._audio_data + + def setValue(self, value: NDArray[np.int16]) -> None: + if not isinstance(value, np.ndarray): + raise TypeError("value must be a numpy array") + self._label._audio_data = value + self._label.update() + self.valueChanged.emit(value) + + def rate(self) -> int: + return self._rate + + def setRate(self, value: int) -> None: + if self._recoding: + raise RuntimeError("Cannot change rate while recording") + self._rate = value + self._label._chunksize = self._rate // 100 + + def resizeEvent(self, a0: QtGui.QResizeEvent) -> None: + self._label.resized.emit() + return super().resizeEvent(a0) + + +class _AudioRecorder(QBaseValueWidget): + _qwidget: QAudioRecorder + + def __init__(self, **kwargs): + super().__init__(QAudioRecorder, "value", "setValue", "valueChanged", **kwargs) + + +@merge_super_sigs +class AudioRecorder(ValueWidget): + """ + A widget for recording microphone input. + + Parameters + ---------- + value : array + 1D array of audio data. + """ + + def __init__(self, **kwargs): + app = use_app() + assert app.native + kwargs["widget_type"] = _AudioRecorder + super().__init__(**kwargs) From 389366172aa9488c1807077f5332cafbf2c9ce59 Mon Sep 17 00:00:00 2001 From: Hanjin Liu Date: Tue, 14 Mar 2023 10:55:23 +0900 Subject: [PATCH 3/5] add clear button --- magicclass/ext/pyaudio/__init__.py | 10 +++++++++- magicclass/ext/pyaudio/widget.py | 31 +++++++++++++++++++----------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/magicclass/ext/pyaudio/__init__.py b/magicclass/ext/pyaudio/__init__.py index 0a585f9f..5c0007fe 100644 --- a/magicclass/ext/pyaudio/__init__.py +++ b/magicclass/ext/pyaudio/__init__.py @@ -1,3 +1,11 @@ from .widget import AudioRecorder +from typing import NewType +import numpy as np +from numpy.typing import NDArray +from magicgui import register_type -__all__ = ["AudioRecorder"] +__all__ = ["AudioRecorder", "AudioData"] + +AudioData = NewType("AudioData", NDArray[np.int16]) + +register_type(AudioData, widget_type=AudioRecorder) diff --git a/magicclass/ext/pyaudio/widget.py b/magicclass/ext/pyaudio/widget.py index 6061f8bf..cafe5fc4 100644 --- a/magicclass/ext/pyaudio/widget.py +++ b/magicclass/ext/pyaudio/widget.py @@ -12,6 +12,8 @@ class QAudioImage(QtW.QLabel): + """The (auto-updating) image of the audio data.""" + resized = Signal() def __init__(self, parent: QtW.QWidget | None = None): @@ -42,17 +44,21 @@ def update(self) -> None: def _image_array(self) -> NDArray[np.uint8]: w, h = self.width(), self.height() audio = _max_binning(self._audio_data, self._chunksize, w * 2) + nh = h // 2 + sigmax = 2**16 half_array = np.stack( - [np.linspace(0, 2**16 - 1, h // 2)] * audio.size, axis=1 + [np.linspace(sigmax / nh, sigmax, nh, endpoint=False)] * audio.size, + axis=1, ) array = np.concatenate([half_array[::-1], half_array], axis=0) binary = array < audio - image = np.full(array.shape + (4,), 248, dtype=np.uint8) + image = np.full(array.shape + (4,), 240, dtype=np.uint8) image[binary] = np.array([0, 0, 255, 255], dtype=np.uint8)[np.newaxis] return image def _max_binning(arr: NDArray[np.int16], n: int, width: int) -> NDArray[np.int16]: + """Bin the input 1D data, clipped by the given width.""" nchunks, res = divmod(arr.size, n) if nchunks < width: out_left = np.abs(arr[:-res]).reshape(-1, n).max(axis=1) @@ -86,16 +92,22 @@ def __init__(self, parent: QtW.QWidget | None = None) -> None: self._recoding = False def _setup_ui(self): - self._btn = QtW.QPushButton("Rec") - self._btn.setFixedSize(36, 24) + self._btn_rec = QtW.QPushButton("Rec") + self._btn_rec.setFixedSize(28, 24) + self._btn_clear = QtW.QPushButton("Clear") + self._btn_rec.setFixedSize(36, 24) self._label = QAudioImage() layout = QtW.QHBoxLayout() layout.setContentsMargins(0, 0, 0, 0) self.setLayout(layout) layout.addWidget(self._label) - layout.addWidget(self._btn) + layout.addWidget(self._btn_rec) + layout.addWidget(self._btn_clear) - self._btn.clicked.connect(self._on_button_clicked) + self._btn_rec.clicked.connect(lambda: self.setRecording(not self.recording())) + self._btn_clear.clicked.connect( + lambda: self.setValue(np.zeros(0, dtype=np.int16)) + ) def _update_data(self): _incoming = self._read_input() @@ -106,9 +118,6 @@ def _read_input(self): ret = np.frombuffer(ret_bytes, dtype=np.int16) return ret - def _on_button_clicked(self): - self.setRecording(not self.recording()) - def recording(self) -> bool: return self._recoding @@ -117,10 +126,10 @@ def setRecording(self, value: bool) -> None: self._recoding = value if self._recoding and not was_recording: self._timer.start(10) - self._btn.setText("Stop") + self._btn_rec.setText("Stop") elif not self._recoding and was_recording: self._timer.stop() - self._btn.setText("Rec") + self._btn_rec.setText("Rec") def value(self) -> NDArray[np.int16]: return self._label._audio_data From 40424b77fc9a97edcd5dd27480e22d53d5bc096d Mon Sep 17 00:00:00 2001 From: Hanjin Liu Date: Wed, 15 Mar 2023 12:19:34 +0900 Subject: [PATCH 4/5] minor fix --- magicclass/ext/pyaudio/widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicclass/ext/pyaudio/widget.py b/magicclass/ext/pyaudio/widget.py index cafe5fc4..eeaa7ba8 100644 --- a/magicclass/ext/pyaudio/widget.py +++ b/magicclass/ext/pyaudio/widget.py @@ -95,7 +95,7 @@ def _setup_ui(self): self._btn_rec = QtW.QPushButton("Rec") self._btn_rec.setFixedSize(28, 24) self._btn_clear = QtW.QPushButton("Clear") - self._btn_rec.setFixedSize(36, 24) + self._btn_clear.setFixedSize(34, 24) self._label = QAudioImage() layout = QtW.QHBoxLayout() layout.setContentsMargins(0, 0, 0, 0) From ebe83b414b0330c624bc790514f07575e77424b5 Mon Sep 17 00:00:00 2001 From: Hanjin Liu Date: Sat, 8 Apr 2023 17:38:57 +0900 Subject: [PATCH 5/5] add doc --- magicclass/ext/pyaudio/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/magicclass/ext/pyaudio/__init__.py b/magicclass/ext/pyaudio/__init__.py index 5c0007fe..e3c79684 100644 --- a/magicclass/ext/pyaudio/__init__.py +++ b/magicclass/ext/pyaudio/__init__.py @@ -9,3 +9,16 @@ AudioData = NewType("AudioData", NDArray[np.int16]) register_type(AudioData, widget_type=AudioRecorder) + +__doc__ = """ +An extension submodule for audio input. + +Examples +-------- + +>>> from magicgui import magicgui +>>> from magicclass.ext.pyaudio import AudioData +>>> @magicgui +>>> def foo(audio: AudioData): +... print(audio.shape) +"""