src/imgui/ImGuiWaveViewer.cc at main · doomsdayone/openmsx

◆ main openmsx / src / imgui / ImGuiWaveViewer.cc

17.8 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw

#include "ImGuiWaveViewer.hh"

#include "ImGuiPlot.hh"

#include "MSXMixer.hh"
#include "MSXMotherBoard.hh"
#include "SoundDevice.hh"

#include "FFTReal.hh"
#include "fast_log2.hh"
#include "halfband.hh"
#include "hammingWindow.hh"
#include "narrow.hh"
#include "ranges.hh"
#include "xrange.hh"
#include "yin.hh"

#include "imgui.h"
#include "imgui_internal.h" // for ImLerp

#include <algorithm>
#include <cmath>
#include <functional>
#include <numbers>
#include <numeric>
#include <ranges>

namespace openmsx {

#define SHOW_CHIP_KEY "showChip"

void ImGuiWaveViewer::save(ImGuiTextBuffer& buf)
{
	savePersistent(buf, *this, persistentElements);
	for (const auto& chip: openChips) {
		buf.appendf(SHOW_CHIP_KEY"=%s\n", chip.c_str());
	}
}

void ImGuiWaveViewer::loadStart()
{
	openChips.clear();
}

void ImGuiWaveViewer::loadLine(std::string_view name, zstring_view value)
{
	if (loadOnePersistent(name, value, *this, persistentElements)) {
		// already handled
	} else if (name == SHOW_CHIP_KEY) {
		openChips.emplace(value);
	}
}

static void paintVUMeter(std::span<const float>& buf, float factor, bool muted)
{
	// skip if not visible
	gl::vec2 pos = ImGui::GetCursorScreenPos();
	ImGui::SetNextItemWidth(-FLT_MIN); // full cell-width
	auto width = ImGui::CalcItemWidth();
	auto height = ImGui::GetFrameHeight();
	ImGui::Dummy(gl::vec2{width, height});
	if (!ImGui::IsItemVisible()) return;
	if (buf.size() <= 2) return;

	// calculate the average power of the signal
	auto len = float(buf.size());
	auto avg = std::reduce(buf.begin(), buf.end()) / len;
	auto squaredSum = std::transform_reduce(buf.begin(), buf.end(), 0.0f,
		std::plus<>{}, [avg](float x) { auto norm = x - avg; return norm * norm; });
	if (squaredSum == 0.0f) {
		buf = {}; // allows to skip waveform and spectrum calculations
		return;
	}
	auto power = fast_log2((squaredSum * factor * factor) / len);

	// transform into a value for how to draw this [0, 1]
	static constexpr auto convertLog = std::numbers::ln10_v<float> / std::numbers::ln2_v<float>; // log2 vs log10
	static constexpr auto range = 6.0f * convertLog; // 60dB
	auto clamped = std::clamp(power, -range, 0.0f);
	auto f = (clamped + range) / range;

	// draw gradient
	auto size = gl::vec2{f * width, height};
	auto colorL = muted ? ImVec4{0.2f, 0.2f, 0.2f, 1.0f} : ImVec4{0.0f, 1.0f, 0.0f, 1.0f}; // green
	auto colorR = muted ? ImVec4{0.7f, 0.7f, 0.7f, 1.0f} : ImVec4{1.0f, 0.0f, 0.0f, 1.0f}; // red
	auto color1 = ImGui::ColorConvertFloat4ToU32(colorL);
	auto color2 = ImGui::ColorConvertFloat4ToU32(ImLerp(colorL, colorR, f));
	auto* drawList = ImGui::GetWindowDrawList();
	drawList->AddRectFilledMultiColor(
		pos, pos + size,
		color1, color2, color2, color1);
}

static void paintWave(std::span<const float> buf)
{
	// skip if not visible
	auto pos = ImGui::GetCursorPos();
	ImGui::SetNextItemWidth(-FLT_MIN); // full cell-width
	auto size = gl::vec2{ImGui::CalcItemWidth(), ImGui::GetFrameHeight()};
	ImGui::Dummy(size);
	if (!ImGui::IsItemVisible()) return;
	ImGui::SetCursorPos(pos);

	if (buf.size() < 2) {
		// otherwise PlotLines() doesn't draw anything, we want a line in the middle
		static constexpr std::array<float, 2> silent = {0.0f, 0.0f};
		buf = silent;
	}
	auto peak = std::transform_reduce(buf.begin(), buf.end(), 0.0f,
		[](auto x, auto y) { return std::max(x, y); },
		[](auto x) { return std::abs(x); });
	if (peak == 0.0f) peak = 1.0f; // force line in the middle

	plotLines(buf, -peak, peak, size);
}

struct ReduceResult {
	std::span<float> result; // down-sampled input
	std::span<float> extendedResult; // zero-padded to power-of-2
	float normalize = 1.0f; // down-sampling changes amplitude
	float reducedSampleRate = 0.0f;
};
static ReduceResult reduce(std::span<const float> buf, std::span<float> work, size_t fftLen, float sampleRate)
{
	// allocate part of the workBuffer (like a monotonic memory allocator)
	auto allocate = [&](size_t size) {
		assert(size <= work.size());
		auto result = work.first(size);
		work = work.subspan(size);
		return result;
	};

	// This function reduces (or rarely extends) the given buffer to exactly
	// 2048 samples, while retaining the lower part of the frequency
	// spectrum.
	//
	// Typically 'buf' contains too many samples. E.g. the PSG produces
	// sound at ~224kHz, so 1/7 second takes ~31.9k samples. We're not
	// interested in those very high frequencies. And we don't want to run a
	// needlessly expensive FFT operation. Therefor this routine:
	// * Filters away the upper-half of the frequency spectrum (via a
	//   half-band FIR filter).
	// * Then drops every other sample (this doesn't change the spectrum if
	//   the filter has high enough quality).
	// * Repeat until the buffer has 'fftLen' samples or less.
	// * Finally zero-pad the result to exactly 'fftLen' samples (this does not
	//   change the spectrum).
	float normalize = 1.0f;
	std::span<float> extended;
	if (buf.size() <= fftLen) {
		extended = allocate(fftLen);
		auto buf2 = extended.subspan(0, buf.size());
		copy_to_range(buf, buf2);
		buf = buf2;
	} else {
		assert(buf.size() >= HALF_BAND_EXTRA);
		extended = allocate(std::max((buf.size() - HALF_BAND_EXTRA) / 2, fftLen));
		do {
			static_assert(HALF_BAND_EXTRA & 1);
			if ((buf.size() & 1) == 0) {
				buf = buf.subspan(1); // drop oldest sample (need an odd number)
			}
			assert(buf.size() >= HALF_BAND_EXTRA);
			auto buf2 = extended.subspan(0, (buf.size() - HALF_BAND_EXTRA) / 2);

			halfBand(buf, buf2); // possibly inplace
			normalize *= 0.5f;
			sampleRate *= 0.5f;

			buf = buf2;
		} while (buf.size() > fftLen);
		extended = extended.subspan(0, fftLen);
	}
	std::ranges::fill(extended.subspan(buf.size()), 0.0f);
	auto result = extended.subspan(0, buf.size());
	return {.result = result,
		.extendedResult = extended,
		.normalize = normalize,
		.reducedSampleRate = sampleRate};
}

// format with "Hz" or "kHz" suffix and 3 significant digits
[[nodiscard]] static std::string formatFreq(int freq, int noteNum)
{
	std::array<char, 4> noteBuf = {' ', ' ', ' ', ' '};
	(void)noteNum2str(noteNum, std::span<char, 3>(noteBuf.data(), 3));
	std::string_view note{noteBuf.data(), 4};

	if (freq < 1000) {
		return strCat(note, freq, "Hz");
	} else {
		auto k = freq / 1000;
		auto t = (freq % 1000) / 10;
		char t1 = char(t / 10) + '0';
		char t2 = char(t % 10) + '0';
		return strCat(note, k, '.', t1, t2, "kHz");
	}
}
[[nodiscard]] static std::string formatFreq(float freq)
{
	return formatFreq(int(std::lround(freq)), freq2noteNum(freq));
}

static void paintSpectrum(std::span<const float> buf, float factor, const SoundDevice& device)
{
	static constexpr auto convertLog = std::numbers::ln10_v<float> / std::numbers::ln2_v<float>; // log2 vs log10
	static constexpr auto range = 5.0f * convertLog; // 50dB

	// skip if not visible
	auto pos = ImGui::GetCursorPos();
	ImGui::SetNextItemWidth(-FLT_MIN); // full cell-width
	auto size = gl::vec2{ImGui::CalcItemWidth(), ImGui::GetFrameHeight()};
	ImGui::Dummy(size);
	if (!ImGui::IsItemVisible()) return;
	ImGui::SetCursorPos(pos);

	const auto& style = ImGui::GetStyle();
	auto graphWidth = size.x - 2.0f * style.FramePadding.x;
	auto fftLen = std::clamp<size_t>(2 * std::bit_ceil(size_t(graphWidth)), 256, 2048);
	switch (fftLen) {
	case 256: // 128 pixels-wide or less
		buf = buf.last(buf.size() / 8); // keep last 1/8 (18ms, 56Hz bins)
		break;
	case 512: // 129..256 pixels (this is the default)
		buf = buf.last(buf.size() / 4); // keep last 1/4 (36ms, 28Hz bins)
		break;
	case 1024: // 257..512 pixels
		buf = buf.last(buf.size() / 2); // keep last 1/2 (71ms, 14Hz bins)
		break;
	default: // more than 513 pixels wide
		// keep full buffer (143ms, 7Hz bins)
		break;
	}

	float sampleRate = 0.0f;
	std::array<float, 1023> magnitude_; // ok, uninitialized
	std::span<float> magnitude;
	if (buf.size() >= 2) {
		// We want to take an FFT of fixed length (256, 512, 1024, 2048 points), reduce the
		// input (decimate + zero-pad) to this exact length.
		std::array<float, 32768> workBuf; // ok, uninitialized
		auto [signal, zeroPadded, normalize, sampleRate_] = reduce(buf, workBuf, fftLen, device.getNativeSampleRate());
		sampleRate = sampleRate_;
		assert(zeroPadded.size() == fftLen);

		// remove DC and apply window-function
		auto window = hammingWindow(narrow<unsigned>(signal.size()));
		auto avg = std::reduce(signal.begin(), signal.end()) / float(signal.size());
		for (auto [s, w] : std::views::zip(signal, window)) {
			s = (s - avg) * w;
		}

		// perform FFT
		std::array<float, 2048> tmp_; // ok, uninitialized
		std::array<float, 2048> f_;   // ok, uninitialized
		switch (fftLen) {
		case 256:
			FFTReal<8>::execute(subspan<256>(zeroPadded), subspan<256>(f_), subspan<256>(tmp_));
			break;
		case 512:
			FFTReal<9>::execute(subspan<512>(zeroPadded), subspan<512>(f_), subspan<512>(tmp_));
			break;
		case 1024:
			FFTReal<10>::execute(subspan<1024>(zeroPadded), subspan<1024>(f_), subspan<1024>(tmp_));
			break;
		default:
			FFTReal<11>::execute(subspan<2048>(zeroPadded), subspan<2048>(f_), subspan<2048>(tmp_));
			break;
		}
		auto f = subspan(f_, 0, fftLen);

		// combine real and imaginary components into magnitude (we ignore phase)
		normalize *= factor;
		auto offset = fast_log2(normalize * normalize * (1.0f / float(fftLen))) + range;
		auto halfFftLen = fftLen / 2;
		magnitude = subspan(magnitude_, 0, halfFftLen - 1);
		for (unsigned i = 0; i < halfFftLen - 1; ++i) {
			float real = f[i + 1];
			float imag = f[i + 1 + halfFftLen];
			float mag = real * real + imag * imag;
			magnitude[i] = fast_log2(mag) + offset;
		}
	}

	// actually plot the result
	plotHistogram(magnitude, 0.0f, range, size);

	simpleToolTip([&] -> std::string {
		auto scrnPosX = ImGui::GetCursorScreenPos().x + style.FramePadding.x;
		auto mouseX = (ImGui::GetIO().MousePos.x - scrnPosX) / graphWidth;
		if ((mouseX <= 0.0f) || (mouseX >= 1.0f)) return {};

		if (sampleRate == 0.0f) {
			// silent -> reduced sampleRate hasn't been calculated yet
			sampleRate = device.getNativeSampleRate();
			auto samples = device.getLastMonoBufferSize();
			switch (fftLen) {
				case 256:  samples /= 8; break;
				case 512:  samples /= 4; break;
				case 1024: samples /= 2; break;
				default: /*nothing*/     break;
			}
			while (samples > fftLen) {
				sampleRate *= 0.5f;
				if ((samples & 1) == 0) --samples;
				samples = (samples - HALF_BAND_EXTRA) / 2;
			}
		}

		return formatFreq(sampleRate * 0.5f * mouseX);
	});
}

[[nodiscard]] static yin::PitchResult calcPitch(std::span<const float> buf, const SoundDevice& device)
{
	if (buf.size() < 512) return {};

	auto sampleRate = device.getNativeSampleRate();

	std::array<float, 32768> workBuf;
	static constexpr float MAX_RATE = 22'000.0f; // 22kHz
	while (sampleRate > MAX_RATE) {
		static_assert(HALF_BAND_EXTRA & 1);
		if ((buf.size() & 1) == 0) {
			buf = buf.subspan(1); // drop oldest sample (need an odd number)
		}
		assert(buf.size() >= HALF_BAND_EXTRA);

		auto outSize = (buf.size() - HALF_BAND_EXTRA) / 2;
		assert(outSize <= workBuf.size());
		auto out = std::span{workBuf}.subspan(0, outSize);

		halfBand(buf, out); // possibly inplace

		sampleRate *= 0.5f;
		buf = out;
	}
	// at this point sampleRate is between 11kHz and 22kHz
	if (buf.size() < 512) return {}; // need minimum size for good detection

	return yin::detectPitch(buf, sampleRate);
}

static void paintPitch(std::span<const float> buf, const SoundDevice& device)
{
	auto [freq, err] = calcPitch(buf, device);
	auto noteNum = freq2noteNum(freq);

	auto iFreq = int(std::lround(freq));
	auto pos = ImGui::GetCursorPos();
	ImGui::InvisibleButton("hover", {-FLT_MIN, ImGui::GetFrameHeight()});
	simpleToolTip([&]{
		auto f = (err > 0.8) ? 0 : iFreq;
		return strCat(formatFreq(f, noteNum), "\nconfidence=", 1.0f - err);
	});
	ImGui::SetCursorPos(pos);

	using P = gl::vec2;
	using Rect = std::array<gl::vec2, 2>;
	using Rect2 = std::array<Rect, 2>;
	struct K { ImU32 col = 0; Rect2 r2; };
	static constexpr ImU32 white = 0xffffffff;
	static constexpr ImU32 black = 0xff000000;
	static constexpr ImU32 gray  = 0xff808080; // background
	static constexpr ImU32 press = 0xff0000ff; // pressed key
	std::array<K, 12> keys = {
		K{.col = white, .r2 = {Rect{P{ 0, 0}, P{ 8, 13}}, Rect{P{ 0, 13}, P{12, 19}}}}, // C
		K{.col = black, .r2 = {Rect{P{ 8, 0}, P{18, 13}}                            }}, // C#
		K{.col = white, .r2 = {Rect{P{18, 0}, P{22, 13}}, Rect{P{14, 13}, P{26, 19}}}}, // D
		K{.col = black, .r2 = {Rect{P{22, 0}, P{32, 13}}                            }}, // D#
		K{.col = white, .r2 = {Rect{P{32, 0}, P{40, 13}}, Rect{P{28, 13}, P{40, 19}}}}, // E
		K{.col = white, .r2 = {Rect{P{42, 0}, P{50, 13}}, Rect{P{42, 13}, P{54, 19}}}}, // F
		K{.col = black, .r2 = {Rect{P{50, 0}, P{60, 13}}                            }}, // F#
		K{.col = white, .r2 = {Rect{P{60, 0}, P{64, 13}}, Rect{P{56, 13}, P{68, 19}}}}, // G
		K{.col = black, .r2 = {Rect{P{64, 0}, P{74, 13}}                            }}, // G#
		K{.col = white, .r2 = {Rect{P{74, 0}, P{78, 13}}, Rect{P{70, 13}, P{82, 19}}}}, // A
		K{.col = black, .r2 = {Rect{P{78, 0}, P{88, 13}}                            }}, // A#
		K{.col = white, .r2 = {Rect{P{88, 0}, P{94, 13}}, Rect{P{84, 13}, P{94, 19}}}}, // B
	};
	auto tl = ImGui::GetCursorScreenPos();
	auto h = ImGui::GetFrameHeight();
	auto trScale = h * (1.0f / 19.0f);
	auto tr = [&](gl::vec2 p) { return tl + p * trScale; };

	std::optional<int> noteInOctave = (err < 0.15f) ? (noteNum % 12) : std::optional<int>{};
	auto* drawList = ImGui::GetWindowDrawList();
	drawList->AddRectFilled(tr({0, 0}), tr({94, 19}), gray);
	for (auto i : xrange(12)) {
		const auto& key = keys[i];
		auto col = (i == noteInOctave) ? press : key.col;
		const auto& r0 = key.r2[0];
		drawList->AddRectFilled(tr(r0[0]), tr(r0[1]), col);
		const auto& r1 = key.r2[1];
		if (r1[0].y != 0.0f) {
			drawList->AddRectFilled(tr(r1[0]), tr(r1[1]), col);
		}
	}
	const auto& style = ImGui::GetStyle();
	ImGui::Dummy({94 * trScale, h});
	ImGui::SameLine();

	if (err > 0.2f) return; // no clear frequency, don't draw anything
	auto colText = style.Colors[ImGuiCol_Text];
	auto colTextDisabled = style.Colors[ImGuiCol_TextDisabled];
	auto col = [&]{
		if (err < 0.15f) {
			return ImLerp(colText, colTextDisabled, err * (1.0f / 0.15f));
		} else {
			auto c = colTextDisabled;
			c.w = std::lerp(c.w, 0.0f, (err - 0.15f) * (1.0f / (0.20f - 0.15f)));
			return c;
		}
	}();
	ImGui::AlignTextToFramePadding();
	im::StyleColor(ImGuiCol_Text, col, [&]{
		ImGui::TextUnformatted(formatFreq(iFreq, noteNum));
	});
}

static void stereoToMono(std::span<const float> stereo, float factorL, float factorR,
                         std::vector<float>& mono)
{
	assert((stereo.size() & 1) == 0);
	auto size = stereo.size() / 2;
	mono.resize(size);
	for (auto i : xrange(size)) {
		mono[i] = factorL * stereo[2 * i + 0]
		        + factorR * stereo[2 * i + 1];
	}
}

static void paintDevice(SoundDevice& device, std::span<const MSXMixer::SoundDeviceInfo::ChannelSettings> settings)
{
	static std::vector<float> tmpBuf; // recycle buffer for all channels, for all frames

	bool stereo = device.hasStereoChannels();
	auto [factorL, factorR] = device.getAmplificationFactor();
	auto factor = stereo ? 1.0f : factorL;

	im::ID_for_range(device.getNumChannels(), [&](int channel) {
		auto monoBuf = [&]{
			auto buf = device.getLastBuffer(channel);
			if (!stereo) return buf;
			stereoToMono(buf, factorL, factorR, tmpBuf);
			return std::span<const float>{tmpBuf};
		}();

		if (ImGui::TableNextColumn()) { // name
			ImGui::StrCat(channel + 1);
		}
		auto& muteSetting = *settings[channel].mute;
		bool muted = muteSetting.getBoolean();
		if (ImGui::TableNextColumn()) { // mute
			if (ImGui::Checkbox("##mute", &muted)) {
				muteSetting.setBoolean(muted);
			}
		}
		if (ImGui::TableNextColumn()) { // vu-meter
			paintVUMeter(monoBuf, factor, muted);
		}
		if (ImGui::TableNextColumn()) { // waveform
			paintWave(monoBuf);
		}
		if (ImGui::TableNextColumn()) { // spectrum
			paintSpectrum(monoBuf, factor, device);
		}
		if (ImGui::TableNextColumn()) { // pitch estimation
			paintPitch(monoBuf, device);
		}
	});
}

void ImGuiWaveViewer::paint(MSXMotherBoard* motherBoard)
{
	if (!show || !motherBoard) return;

	ImGui::SetNextWindowSize(gl::vec2{38, 15} * ImGui::GetFontSize(), ImGuiCond_FirstUseEver);
	im::Window("Audio channel viewer", &show, [&]{
		for (const auto& info: motherBoard->getMSXMixer().getDeviceInfos()) {
			auto& device = *info.device;
			const auto& name = device.getName();
			auto it = openChips.find(name);
			bool wasOpen = it != openChips.end();
			bool nowOpen = ImGui::CollapsingHeader(name.c_str(), wasOpen ? ImGuiTreeNodeFlags_DefaultOpen : ImGuiTreeNodeFlags_None);
			if (wasOpen != nowOpen) {
			    if (nowOpen) openChips.insert(name); else openChips.erase(it);
			}
			if (!nowOpen) continue;
			HelpMarker("Right-click column header to (un)hide columns.\n"
			           "Drag to reorder or resize columns.");

			int flags = ImGuiTableFlags_RowBg |
			            ImGuiTableFlags_BordersV |
			            ImGuiTableFlags_BordersOuter |
			            ImGuiTableFlags_Resizable |
			            ImGuiTableFlags_Reorderable |
			            ImGuiTableFlags_Hideable |
			            ImGuiTableFlags_SizingStretchProp;
			im::Table("##table", 6, flags, [&]{ // note: use the same id for all tables
				ImGui::TableSetupScrollFreeze(0, 1); // Make top row always visible
				ImGui::TableSetupColumn("ch.", ImGuiTableColumnFlags_NoReorder | ImGuiTableColumnFlags_NoResize | ImGuiTableColumnFlags_WidthFixed);
				// TODO: make this work here, or add an alternative: simpleToolTip("channel number");
				ImGui::TableSetupColumn("mute", ImGuiTableColumnFlags_DefaultHide | ImGuiTableColumnFlags_NoResize | ImGuiTableColumnFlags_WidthFixed);
				ImGui::TableSetupColumn("VU-meter", 0, 1.0f);
				ImGui::TableSetupColumn("Waveform", 0, 2.0f);
				ImGui::TableSetupColumn("Spectrum", 0, 3.0f);
				ImGui::TableSetupColumn("Pitch estimation", ImGuiTableColumnFlags_DefaultHide, 1.0f);
				ImGui::TableHeadersRow();
				im::ID(name, [&]{
					paintDevice(device, info.channelSettings);
				});
			});
		}
	});
}

} // namespace openmsx

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	#include "ImGuiWaveViewer.hh"
2
3	#include "ImGuiPlot.hh"
4
5	#include "MSXMixer.hh"
6	#include "MSXMotherBoard.hh"
7	#include "SoundDevice.hh"
8
9	#include "FFTReal.hh"
10	#include "fast_log2.hh"
11	#include "halfband.hh"
12	#include "hammingWindow.hh"
13	#include "narrow.hh"
14	#include "ranges.hh"
15	#include "xrange.hh"
16	#include "yin.hh"
17
18	#include "imgui.h"
19	#include "imgui_internal.h" // for ImLerp
20
21	#include <algorithm>
22	#include <cmath>
23	#include <functional>
24	#include <numbers>
25	#include <numeric>
26	#include <ranges>
27
28	namespace openmsx {
29
30	#define SHOW_CHIP_KEY "showChip"
31
32	void ImGuiWaveViewer::save(ImGuiTextBuffer& buf)
33	{
34	savePersistent(buf, *this, persistentElements);
35	for (const auto& chip: openChips) {
36	buf.appendf(SHOW_CHIP_KEY"=%s\n", chip.c_str());
37	}
38	}
39
40	void ImGuiWaveViewer::loadStart()
41	{
42	openChips.clear();
43	}
44
45	void ImGuiWaveViewer::loadLine(std::string_view name, zstring_view value)
46	{
47	if (loadOnePersistent(name, value, *this, persistentElements)) {
48	// already handled
49	} else if (name == SHOW_CHIP_KEY) {
50	openChips.emplace(value);
51	}
52	}
53
54	static void paintVUMeter(std::span<const float>& buf, float factor, bool muted)
55	{
56	// skip if not visible
57	gl::vec2 pos = ImGui::GetCursorScreenPos();
58	ImGui::SetNextItemWidth(-FLT_MIN); // full cell-width
59	auto width = ImGui::CalcItemWidth();
60	auto height = ImGui::GetFrameHeight();
61	ImGui::Dummy(gl::vec2{width, height});
62	if (!ImGui::IsItemVisible()) return;
63	if (buf.size() <= 2) return;
64
65	// calculate the average power of the signal
66	auto len = float(buf.size());
67	auto avg = std::reduce(buf.begin(), buf.end()) / len;
68	auto squaredSum = std::transform_reduce(buf.begin(), buf.end(), 0.0f,
69	std::plus<>{}, [avg](float x) { auto norm = x - avg; return norm * norm; });
70	if (squaredSum == 0.0f) {
71	buf = {}; // allows to skip waveform and spectrum calculations
72	return;
73	}
74	auto power = fast_log2((squaredSum * factor * factor) / len);
75
76	// transform into a value for how to draw this [0, 1]
77	static constexpr auto convertLog = std::numbers::ln10_v<float> / std::numbers::ln2_v<float>; // log2 vs log10
78	static constexpr auto range = 6.0f * convertLog; // 60dB
79	auto clamped = std::clamp(power, -range, 0.0f);
80	auto f = (clamped + range) / range;
81
82	// draw gradient
83	auto size = gl::vec2{f * width, height};
84	auto colorL = muted ? ImVec4{0.2f, 0.2f, 0.2f, 1.0f} : ImVec4{0.0f, 1.0f, 0.0f, 1.0f}; // green
85	auto colorR = muted ? ImVec4{0.7f, 0.7f, 0.7f, 1.0f} : ImVec4{1.0f, 0.0f, 0.0f, 1.0f}; // red
86	auto color1 = ImGui::ColorConvertFloat4ToU32(colorL);
87	auto color2 = ImGui::ColorConvertFloat4ToU32(ImLerp(colorL, colorR, f));
88	auto* drawList = ImGui::GetWindowDrawList();
89	drawList->AddRectFilledMultiColor(
90	pos, pos + size,
91	color1, color2, color2, color1);
92	}
93
94	static void paintWave(std::span<const float> buf)
95	{
96	// skip if not visible
97	auto pos = ImGui::GetCursorPos();
98	ImGui::SetNextItemWidth(-FLT_MIN); // full cell-width
99	auto size = gl::vec2{ImGui::CalcItemWidth(), ImGui::GetFrameHeight()};
100	ImGui::Dummy(size);
101	if (!ImGui::IsItemVisible()) return;
102	ImGui::SetCursorPos(pos);
103
104	if (buf.size() < 2) {
105	// otherwise PlotLines() doesn't draw anything, we want a line in the middle
106	static constexpr std::array<float, 2> silent = {0.0f, 0.0f};
107	buf = silent;
108	}
109	auto peak = std::transform_reduce(buf.begin(), buf.end(), 0.0f,
110	[](auto x, auto y) { return std::max(x, y); },
111	[](auto x) { return std::abs(x); });
112	if (peak == 0.0f) peak = 1.0f; // force line in the middle
113
114	plotLines(buf, -peak, peak, size);
115	}
116
117	struct ReduceResult {
118	std::span<float> result; // down-sampled input
119	std::span<float> extendedResult; // zero-padded to power-of-2
120	float normalize = 1.0f; // down-sampling changes amplitude
121	float reducedSampleRate = 0.0f;
122	};
123	static ReduceResult reduce(std::span<const float> buf, std::span<float> work, size_t fftLen, float sampleRate)
124	{
125	// allocate part of the workBuffer (like a monotonic memory allocator)
126	auto allocate = [&](size_t size) {
127	assert(size <= work.size());
128	auto result = work.first(size);
129	work = work.subspan(size);
130	return result;
131	};
132
133	// This function reduces (or rarely extends) the given buffer to exactly
134	// 2048 samples, while retaining the lower part of the frequency
135	// spectrum.
136	//
137	// Typically 'buf' contains too many samples. E.g. the PSG produces
138	// sound at ~224kHz, so 1/7 second takes ~31.9k samples. We're not
139	// interested in those very high frequencies. And we don't want to run a
140	// needlessly expensive FFT operation. Therefor this routine:
141	// * Filters away the upper-half of the frequency spectrum (via a
142	// half-band FIR filter).
143	// * Then drops every other sample (this doesn't change the spectrum if
144	// the filter has high enough quality).
145	// * Repeat until the buffer has 'fftLen' samples or less.
146	// * Finally zero-pad the result to exactly 'fftLen' samples (this does not
147	// change the spectrum).
148	float normalize = 1.0f;
149	std::span<float> extended;
150	if (buf.size() <= fftLen) {
151	extended = allocate(fftLen);
152	auto buf2 = extended.subspan(0, buf.size());
153	copy_to_range(buf, buf2);
154	buf = buf2;
155	} else {
156	assert(buf.size() >= HALF_BAND_EXTRA);
157	extended = allocate(std::max((buf.size() - HALF_BAND_EXTRA) / 2, fftLen));
158	do {
159	static_assert(HALF_BAND_EXTRA & 1);
160	if ((buf.size() & 1) == 0) {
161	buf = buf.subspan(1); // drop oldest sample (need an odd number)
162	}
163	assert(buf.size() >= HALF_BAND_EXTRA);
164	auto buf2 = extended.subspan(0, (buf.size() - HALF_BAND_EXTRA) / 2);
165
166	halfBand(buf, buf2); // possibly inplace
167	normalize *= 0.5f;
168	sampleRate *= 0.5f;
169
170	buf = buf2;
171	} while (buf.size() > fftLen);
172	extended = extended.subspan(0, fftLen);
173	}
174	std::ranges::fill(extended.subspan(buf.size()), 0.0f);
175	auto result = extended.subspan(0, buf.size());
176	return {.result = result,
177	.extendedResult = extended,
178	.normalize = normalize,
179	.reducedSampleRate = sampleRate};
180	}
181
182	// format with "Hz" or "kHz" suffix and 3 significant digits
183	[[nodiscard]] static std::string formatFreq(int freq, int noteNum)
184	{
185	std::array<char, 4> noteBuf = {' ', ' ', ' ', ' '};
186	(void)noteNum2str(noteNum, std::span<char, 3>(noteBuf.data(), 3));
187	std::string_view note{noteBuf.data(), 4};
188
189	if (freq < 1000) {
190	return strCat(note, freq, "Hz");
191	} else {
192	auto k = freq / 1000;
193	auto t = (freq % 1000) / 10;
194	char t1 = char(t / 10) + '0';
195	char t2 = char(t % 10) + '0';
196	return strCat(note, k, '.', t1, t2, "kHz");
197	}
198	}
199	[[nodiscard]] static std::string formatFreq(float freq)
200	{
201	return formatFreq(int(std::lround(freq)), freq2noteNum(freq));
202	}
203
204	static void paintSpectrum(std::span<const float> buf, float factor, const SoundDevice& device)
205	{
206	static constexpr auto convertLog = std::numbers::ln10_v<float> / std::numbers::ln2_v<float>; // log2 vs log10
207	static constexpr auto range = 5.0f * convertLog; // 50dB
208
209	// skip if not visible
210	auto pos = ImGui::GetCursorPos();
211	ImGui::SetNextItemWidth(-FLT_MIN); // full cell-width
212	auto size = gl::vec2{ImGui::CalcItemWidth(), ImGui::GetFrameHeight()};
213	ImGui::Dummy(size);
214	if (!ImGui::IsItemVisible()) return;
215	ImGui::SetCursorPos(pos);
216
217	const auto& style = ImGui::GetStyle();
218	auto graphWidth = size.x - 2.0f * style.FramePadding.x;
219	auto fftLen = std::clamp<size_t>(2 * std::bit_ceil(size_t(graphWidth)), 256, 2048);
220	switch (fftLen) {
221	case 256: // 128 pixels-wide or less
222	buf = buf.last(buf.size() / 8); // keep last 1/8 (18ms, 56Hz bins)
223	break;
224	case 512: // 129..256 pixels (this is the default)
225	buf = buf.last(buf.size() / 4); // keep last 1/4 (36ms, 28Hz bins)
226	break;
227	case 1024: // 257..512 pixels
228	buf = buf.last(buf.size() / 2); // keep last 1/2 (71ms, 14Hz bins)
229	break;
230	default: // more than 513 pixels wide
231	// keep full buffer (143ms, 7Hz bins)
232	break;
233	}
234
235	float sampleRate = 0.0f;
236	std::array<float, 1023> magnitude_; // ok, uninitialized
237	std::span<float> magnitude;
238	if (buf.size() >= 2) {
239	// We want to take an FFT of fixed length (256, 512, 1024, 2048 points), reduce the
240	// input (decimate + zero-pad) to this exact length.
241	std::array<float, 32768> workBuf; // ok, uninitialized
242	auto [signal, zeroPadded, normalize, sampleRate_] = reduce(buf, workBuf, fftLen, device.getNativeSampleRate());
243	sampleRate = sampleRate_;
244	assert(zeroPadded.size() == fftLen);
245
246	// remove DC and apply window-function
247	auto window = hammingWindow(narrow<unsigned>(signal.size()));
248	auto avg = std::reduce(signal.begin(), signal.end()) / float(signal.size());
249	for (auto [s, w] : std::views::zip(signal, window)) {
250	s = (s - avg) * w;
251	}
252
253	// perform FFT
254	std::array<float, 2048> tmp_; // ok, uninitialized
255	std::array<float, 2048> f_; // ok, uninitialized
256	switch (fftLen) {
257	case 256:
258	FFTReal<8>::execute(subspan<256>(zeroPadded), subspan<256>(f_), subspan<256>(tmp_));
259	break;
260	case 512:
261	FFTReal<9>::execute(subspan<512>(zeroPadded), subspan<512>(f_), subspan<512>(tmp_));
262	break;
263	case 1024:
264	FFTReal<10>::execute(subspan<1024>(zeroPadded), subspan<1024>(f_), subspan<1024>(tmp_));
265	break;
266	default:
267	FFTReal<11>::execute(subspan<2048>(zeroPadded), subspan<2048>(f_), subspan<2048>(tmp_));
268	break;
269	}
270	auto f = subspan(f_, 0, fftLen);
271
272	// combine real and imaginary components into magnitude (we ignore phase)
273	normalize *= factor;
274	auto offset = fast_log2(normalize * normalize * (1.0f / float(fftLen))) + range;
275	auto halfFftLen = fftLen / 2;
276	magnitude = subspan(magnitude_, 0, halfFftLen - 1);
277	for (unsigned i = 0; i < halfFftLen - 1; ++i) {
278	float real = f[i + 1];
279	float imag = f[i + 1 + halfFftLen];
280	float mag = real * real + imag * imag;
281	magnitude[i] = fast_log2(mag) + offset;
282	}
283	}
284
285	// actually plot the result
286	plotHistogram(magnitude, 0.0f, range, size);
287
288	simpleToolTip([&] -> std::string {
289	auto scrnPosX = ImGui::GetCursorScreenPos().x + style.FramePadding.x;
290	auto mouseX = (ImGui::GetIO().MousePos.x - scrnPosX) / graphWidth;
291	if ((mouseX <= 0.0f) \|\| (mouseX >= 1.0f)) return {};
292
293	if (sampleRate == 0.0f) {
294	// silent -> reduced sampleRate hasn't been calculated yet
295	sampleRate = device.getNativeSampleRate();
296	auto samples = device.getLastMonoBufferSize();
297	switch (fftLen) {
298	case 256: samples /= 8; break;
299	case 512: samples /= 4; break;
300	case 1024: samples /= 2; break;
301	default: /nothing/ break;
302	}
303	while (samples > fftLen) {
304	sampleRate *= 0.5f;
305	if ((samples & 1) == 0) --samples;
306	samples = (samples - HALF_BAND_EXTRA) / 2;
307	}
308	}
309
310	return formatFreq(sampleRate * 0.5f * mouseX);
311	});
312	}
313
314	[[nodiscard]] static yin::PitchResult calcPitch(std::span<const float> buf, const SoundDevice& device)
315	{
316	if (buf.size() < 512) return {};
317
318	auto sampleRate = device.getNativeSampleRate();
319
320	std::array<float, 32768> workBuf;
321	static constexpr float MAX_RATE = 22'000.0f; // 22kHz
322	while (sampleRate > MAX_RATE) {
323	static_assert(HALF_BAND_EXTRA & 1);
324	if ((buf.size() & 1) == 0) {
325	buf = buf.subspan(1); // drop oldest sample (need an odd number)
326	}
327	assert(buf.size() >= HALF_BAND_EXTRA);
328
329	auto outSize = (buf.size() - HALF_BAND_EXTRA) / 2;
330	assert(outSize <= workBuf.size());
331	auto out = std::span{workBuf}.subspan(0, outSize);
332
333	halfBand(buf, out); // possibly inplace
334
335	sampleRate *= 0.5f;
336	buf = out;
337	}
338	// at this point sampleRate is between 11kHz and 22kHz
339	if (buf.size() < 512) return {}; // need minimum size for good detection
340
341	return yin::detectPitch(buf, sampleRate);
342	}
343
344	static void paintPitch(std::span<const float> buf, const SoundDevice& device)
345	{
346	auto [freq, err] = calcPitch(buf, device);
347	auto noteNum = freq2noteNum(freq);
348
349	auto iFreq = int(std::lround(freq));
350	auto pos = ImGui::GetCursorPos();
351	ImGui::InvisibleButton("hover", {-FLT_MIN, ImGui::GetFrameHeight()});
352	simpleToolTip([&]{
353	auto f = (err > 0.8) ? 0 : iFreq;
354	return strCat(formatFreq(f, noteNum), "\nconfidence=", 1.0f - err);
355	});
356	ImGui::SetCursorPos(pos);
357
358	using P = gl::vec2;
359	using Rect = std::array<gl::vec2, 2>;
360	using Rect2 = std::array<Rect, 2>;
361	struct K { ImU32 col = 0; Rect2 r2; };
362	static constexpr ImU32 white = 0xffffffff;
363	static constexpr ImU32 black = 0xff000000;
364	static constexpr ImU32 gray = 0xff808080; // background
365	static constexpr ImU32 press = 0xff0000ff; // pressed key
366	std::array<K, 12> keys = {
367	K{.col = white, .r2 = {Rect{P{ 0, 0}, P{ 8, 13}}, Rect{P{ 0, 13}, P{12, 19}}}}, // C
368	K{.col = black, .r2 = {Rect{P{ 8, 0}, P{18, 13}} }}, // C#
369	K{.col = white, .r2 = {Rect{P{18, 0}, P{22, 13}}, Rect{P{14, 13}, P{26, 19}}}}, // D
370	K{.col = black, .r2 = {Rect{P{22, 0}, P{32, 13}} }}, // D#
371	K{.col = white, .r2 = {Rect{P{32, 0}, P{40, 13}}, Rect{P{28, 13}, P{40, 19}}}}, // E
372	K{.col = white, .r2 = {Rect{P{42, 0}, P{50, 13}}, Rect{P{42, 13}, P{54, 19}}}}, // F
373	K{.col = black, .r2 = {Rect{P{50, 0}, P{60, 13}} }}, // F#
374	K{.col = white, .r2 = {Rect{P{60, 0}, P{64, 13}}, Rect{P{56, 13}, P{68, 19}}}}, // G
375	K{.col = black, .r2 = {Rect{P{64, 0}, P{74, 13}} }}, // G#
376	K{.col = white, .r2 = {Rect{P{74, 0}, P{78, 13}}, Rect{P{70, 13}, P{82, 19}}}}, // A
377	K{.col = black, .r2 = {Rect{P{78, 0}, P{88, 13}} }}, // A#
378	K{.col = white, .r2 = {Rect{P{88, 0}, P{94, 13}}, Rect{P{84, 13}, P{94, 19}}}}, // B
379	};
380	auto tl = ImGui::GetCursorScreenPos();
381	auto h = ImGui::GetFrameHeight();
382	auto trScale = h * (1.0f / 19.0f);
383	auto tr = [&](gl::vec2 p) { return tl + p * trScale; };
384
385	std::optional<int> noteInOctave = (err < 0.15f) ? (noteNum % 12) : std::optional<int>{};
386	auto* drawList = ImGui::GetWindowDrawList();
387	drawList->AddRectFilled(tr({0, 0}), tr({94, 19}), gray);
388	for (auto i : xrange(12)) {
389	const auto& key = keys[i];
390	auto col = (i == noteInOctave) ? press : key.col;
391	const auto& r0 = key.r2[0];
392	drawList->AddRectFilled(tr(r0[0]), tr(r0[1]), col);
393	const auto& r1 = key.r2[1];
394	if (r1[0].y != 0.0f) {
395	drawList->AddRectFilled(tr(r1[0]), tr(r1[1]), col);
396	}
397	}
398	const auto& style = ImGui::GetStyle();
399	ImGui::Dummy({94 * trScale, h});
400	ImGui::SameLine();
401
402	if (err > 0.2f) return; // no clear frequency, don't draw anything
403	auto colText = style.Colors[ImGuiCol_Text];
404	auto colTextDisabled = style.Colors[ImGuiCol_TextDisabled];
405	auto col = [&]{
406	if (err < 0.15f) {
407	return ImLerp(colText, colTextDisabled, err * (1.0f / 0.15f));
408	} else {
409	auto c = colTextDisabled;
410	c.w = std::lerp(c.w, 0.0f, (err - 0.15f) * (1.0f / (0.20f - 0.15f)));
411	return c;
412	}
413	}();
414	ImGui::AlignTextToFramePadding();
415	im::StyleColor(ImGuiCol_Text, col, [&]{
416	ImGui::TextUnformatted(formatFreq(iFreq, noteNum));
417	});
418	}
419
420	static void stereoToMono(std::span<const float> stereo, float factorL, float factorR,
421	std::vector<float>& mono)
422	{
423	assert((stereo.size() & 1) == 0);
424	auto size = stereo.size() / 2;
425	mono.resize(size);
426	for (auto i : xrange(size)) {
427	mono[i] = factorL * stereo[2 * i + 0]
428	+ factorR * stereo[2 * i + 1];
429	}
430	}
431
432	static void paintDevice(SoundDevice& device, std::span<const MSXMixer::SoundDeviceInfo::ChannelSettings> settings)
433	{
434	static std::vector<float> tmpBuf; // recycle buffer for all channels, for all frames
435
436	bool stereo = device.hasStereoChannels();
437	auto [factorL, factorR] = device.getAmplificationFactor();
438	auto factor = stereo ? 1.0f : factorL;
439
440	im::ID_for_range(device.getNumChannels(), [&](int channel) {
441	auto monoBuf = [&]{
442	auto buf = device.getLastBuffer(channel);
443	if (!stereo) return buf;
444	stereoToMono(buf, factorL, factorR, tmpBuf);
445	return std::span<const float>{tmpBuf};
446	}();
447
448	if (ImGui::TableNextColumn()) { // name
449	ImGui::StrCat(channel + 1);
450	}
451	auto& muteSetting = *settings[channel].mute;
452	bool muted = muteSetting.getBoolean();
453	if (ImGui::TableNextColumn()) { // mute
454	if (ImGui::Checkbox("##mute", &muted)) {
455	muteSetting.setBoolean(muted);
456	}
457	}
458	if (ImGui::TableNextColumn()) { // vu-meter
459	paintVUMeter(monoBuf, factor, muted);
460	}
461	if (ImGui::TableNextColumn()) { // waveform
462	paintWave(monoBuf);
463	}
464	if (ImGui::TableNextColumn()) { // spectrum
465	paintSpectrum(monoBuf, factor, device);
466	}
467	if (ImGui::TableNextColumn()) { // pitch estimation
468	paintPitch(monoBuf, device);
469	}
470	});
471	}
472
473	void ImGuiWaveViewer::paint(MSXMotherBoard* motherBoard)
474	{
475	if (!show \|\| !motherBoard) return;
476
477	ImGui::SetNextWindowSize(gl::vec2{38, 15} * ImGui::GetFontSize(), ImGuiCond_FirstUseEver);
478	im::Window("Audio channel viewer", &show, [&]{
479	for (const auto& info: motherBoard->getMSXMixer().getDeviceInfos()) {
480	auto& device = *info.device;
481	const auto& name = device.getName();
482	auto it = openChips.find(name);
483	bool wasOpen = it != openChips.end();
484	bool nowOpen = ImGui::CollapsingHeader(name.c_str(), wasOpen ? ImGuiTreeNodeFlags_DefaultOpen : ImGuiTreeNodeFlags_None);
485	if (wasOpen != nowOpen) {
486	if (nowOpen) openChips.insert(name); else openChips.erase(it);
487	}
488	if (!nowOpen) continue;
489	HelpMarker("Right-click column header to (un)hide columns.\n"
490	"Drag to reorder or resize columns.");
491
492	int flags = ImGuiTableFlags_RowBg \|
493	ImGuiTableFlags_BordersV \|
494	ImGuiTableFlags_BordersOuter \|
495	ImGuiTableFlags_Resizable \|
496	ImGuiTableFlags_Reorderable \|
497	ImGuiTableFlags_Hideable \|
498	ImGuiTableFlags_SizingStretchProp;
499	im::Table("##table", 6, flags, [&]{ // note: use the same id for all tables
500	ImGui::TableSetupScrollFreeze(0, 1); // Make top row always visible
501	ImGui::TableSetupColumn("ch.", ImGuiTableColumnFlags_NoReorder \| ImGuiTableColumnFlags_NoResize \| ImGuiTableColumnFlags_WidthFixed);
502	// TODO: make this work here, or add an alternative: simpleToolTip("channel number");
503	ImGui::TableSetupColumn("mute", ImGuiTableColumnFlags_DefaultHide \| ImGuiTableColumnFlags_NoResize \| ImGuiTableColumnFlags_WidthFixed);
504	ImGui::TableSetupColumn("VU-meter", 0, 1.0f);
505	ImGui::TableSetupColumn("Waveform", 0, 2.0f);
506	ImGui::TableSetupColumn("Spectrum", 0, 3.0f);
507	ImGui::TableSetupColumn("Pitch estimation", ImGuiTableColumnFlags_DefaultHide, 1.0f);
508	ImGui::TableHeadersRow();
509	im::ID(name, [&]{
510	paintDevice(device, info.channelSettings);
511	});
512	});
513	}
514	});
515	}
516
517	} // namespace openmsx

Keyboard