Change sound backend from Qt to miniaudio (#4334)

Thanks Greenlandicsmiley, Nerixyz, Yoitsu, and helmak for helping debug & test this

* Remove QMediaPlayer includes

* Prefer local path when generating the sound path

* Update changelog entry number

* Disable pitch & spatialization control

.gitmodules

@@ -35,3 +35,6 @@
 [submodule "lib/googletest"]
 	path = lib/googletest
 	url = https://github.com/google/googletest.git
+[submodule "lib/miniaudio"]
+	path = lib/miniaudio
+	url = https://github.com/mackron/miniaudio.git

CHANGELOG.md

@@ -27,6 +27,7 @@
 - Bugfix: Fixed the split "Search" menu action not opening the correct search window. (#4305)
 - Bugfix: Fixed an issue on Windows when opening links in incognito mode that contained forward slashes in hash (#4307)
 - Bugfix: Fixed an issue where beta versions wouldn't update to stable versions correctly. (#4329)
+- Dev: Changed sound backend from Qt to miniaudio. (#4334)
 - Dev: Remove protocol from QApplication's Organization Domain (so changed from `https://www.chatterino.com` to `chatterino.com`). (#4256)
 - Dev: Ignore `WM_SHOWWINDOW` hide events, causing fewer attempted rescales. (#4198)
 - Dev: Migrated to C++ 20 (#4252, #4257)

lib/miniaudio

@@ -0,0 +1 @@
+Subproject commit c153a947919808419b0bf3f56b6f2ee606d6c5f4

resources/licenses/miniaudio.txt

@@ -0,0 +1,16 @@
+Copyright 2020 David Reid
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

src/Application.cpp

@@ -10,6 +10,7 @@
 #include "controllers/hotkeys/HotkeyController.hpp"
 #include "controllers/ignores/IgnoreController.hpp"
 #include "controllers/notifications/NotificationController.hpp"
+#include "controllers/sound/SoundController.hpp"
 #include "controllers/userdata/UserDataController.hpp"
 #include "debug/AssertInGuiThread.hpp"
 #include "messages/Message.hpp"
@@ -45,6 +46,7 @@
 #include "widgets/splits/Split.hpp"
 #include "widgets/Window.hpp"
 
+#include <miniaudio.h>
 #include <QDesktopServices>
 
 #include <atomic>
@@ -82,6 +84,7 @@ Application::Application(Settings &_settings, Paths &_paths)
     , ffzBadges(&this->emplace<FfzBadges>())
     , seventvBadges(&this->emplace<SeventvBadges>())
     , userData(&this->emplace<UserDataController>())
+    , sound(&this->emplace<SoundController>())
     , logging(&this->emplace<Logging>())
 {
     this->instance = this;

src/Application.hpp

@@ -19,6 +19,7 @@ class HighlightController;
 class HotkeyController;
 class IUserDataController;
 class UserDataController;
+class SoundController;
 
 class Theme;
 class WindowManager;
@@ -92,6 +93,7 @@ public:
     FfzBadges *const ffzBadges{};
     SeventvBadges *const seventvBadges{};
     UserDataController *const userData{};
+    SoundController *const sound{};
 
     /*[[deprecated]]*/ Logging *const logging{};
 

src/CMakeLists.txt

@@ -140,6 +140,9 @@ set(SOURCE_FILES
         controllers/userdata/UserDataController.hpp
         controllers/userdata/UserData.hpp
 
+        controllers/sound/SoundController.cpp
+        controllers/sound/SoundController.hpp
+
         debug/Benchmark.cpp
         debug/Benchmark.hpp
 
@@ -771,6 +774,17 @@ target_include_directories(${LIBRARY_PROJECT} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
 # semver dependency https://github.com/Neargye/semver
 target_include_directories(${LIBRARY_PROJECT} PUBLIC ${CMAKE_SOURCE_DIR}/lib/semver/include)
 
+# miniaudio dependency https://github.com/mackron/miniaudio
+target_include_directories(${LIBRARY_PROJECT} PUBLIC ${CMAKE_SOURCE_DIR}/lib/miniaudio)
+
+if (UNIX)
+    if (CMAKE_DL_LIBS)
+        # libdl is a requirement for miniaudio on Linux
+        message(STATUS "Linking with CMake DL libs: '${CMAKE_DL_LIBS}'")
+        target_link_libraries(${LIBRARY_PROJECT} PUBLIC ${CMAKE_DL_LIBS})
+    endif ()
+endif ()
+
 if (WinToast_FOUND)
     target_link_libraries(${LIBRARY_PROJECT}
         PUBLIC

src/PrecompiledHeader.hpp

@@ -64,7 +64,6 @@
 #    include <QListView>
 #    include <QListWidget>
 #    include <QMap>
-#    include <QMediaPlayer>
 #    include <QMenu>
 #    include <QMessageBox>
 #    include <QMimeData>

src/common/QLogging.cpp

@@ -39,6 +39,7 @@ Q_LOGGING_CATEGORY(chatterinoSettings, "chatterino.settings", logThreshold);
 Q_LOGGING_CATEGORY(chatterinoSeventv, "chatterino.seventv", logThreshold);
 Q_LOGGING_CATEGORY(chatterinoSeventvEventAPI, "chatterino.seventv.eventapi",
                    logThreshold);
+Q_LOGGING_CATEGORY(chatterinoSound, "chatterino.sound", logThreshold);
 Q_LOGGING_CATEGORY(chatterinoStreamerMode, "chatterino.streamermode",
                    logThreshold);
 Q_LOGGING_CATEGORY(chatterinoStreamlink, "chatterino.streamlink", logThreshold);

src/common/QLogging.hpp

@@ -29,6 +29,7 @@ Q_DECLARE_LOGGING_CATEGORY(chatterinoRecentMessages);
 Q_DECLARE_LOGGING_CATEGORY(chatterinoSettings);
 Q_DECLARE_LOGGING_CATEGORY(chatterinoSeventv);
 Q_DECLARE_LOGGING_CATEGORY(chatterinoSeventvEventAPI);
+Q_DECLARE_LOGGING_CATEGORY(chatterinoSound);
 Q_DECLARE_LOGGING_CATEGORY(chatterinoStreamerMode);
 Q_DECLARE_LOGGING_CATEGORY(chatterinoStreamlink);
 Q_DECLARE_LOGGING_CATEGORY(chatterinoTokenizer);

src/controllers/notifications/NotificationController.cpp

@@ -5,6 +5,7 @@
 #include "common/Outcome.hpp"
 #include "common/QLogging.hpp"
 #include "controllers/notifications/NotificationModel.hpp"
+#include "controllers/sound/SoundController.hpp"
 #include "messages/Message.hpp"
 #include "providers/twitch/api/Helix.hpp"
 #include "providers/twitch/TwitchIrcServer.hpp"
@@ -21,7 +22,6 @@
 
 #include <QDesktopServices>
 #include <QDir>
-#include <QMediaPlayer>
 #include <QUrl>
 
 #include <unordered_set>
@@ -97,25 +97,13 @@ void NotificationController::removeChannelNotification(
 }
 void NotificationController::playSound()
 {
-    static auto player = new QMediaPlayer;
-    static QUrl currentPlayerUrl;
-
     QUrl highlightSoundUrl =
         getSettings()->notificationCustomSound
             ? QUrl::fromLocalFile(
                   getSettings()->notificationPathSound.getValue())
             : QUrl("qrc:/sounds/ping2.wav");
 
-    // Set media if the highlight sound url has changed, or if media is buffered
-    if (currentPlayerUrl != highlightSoundUrl ||
-        player->mediaStatus() == QMediaPlayer::BufferedMedia)
-    {
-        player->setMedia(highlightSoundUrl);
-
-        currentPlayerUrl = highlightSoundUrl;
-    }
-
-    player->play();
+    getApp()->sound->play(highlightSoundUrl);
 }
 
 NotificationModel *NotificationController::createModel(QObject *parent,

src/controllers/sound/SoundController.cpp

@@ -0,0 +1,216 @@
+#include "controllers/sound/SoundController.hpp"
+
+#include "common/QLogging.hpp"
+#include "debug/Benchmark.hpp"
+#include "singletons/Paths.hpp"
+#include "singletons/Settings.hpp"
+
+#define MINIAUDIO_IMPLEMENTATION
+#include <miniaudio.h>
+
+#include <limits>
+#include <memory>
+
+namespace chatterino {
+
+// NUM_SOUNDS specifies how many simultaneous default ping sounds & decoders to create
+constexpr const auto NUM_SOUNDS = 4;
+
+SoundController::SoundController()
+    : context(std::make_unique<ma_context>())
+    , resourceManager(std::make_unique<ma_resource_manager>())
+    , device(std::make_unique<ma_device>())
+    , engine(std::make_unique<ma_engine>())
+{
+}
+
+void SoundController::initialize(Settings &settings, Paths &paths)
+{
+    (void)(settings);
+    (void)(paths);
+
+    ma_result result{};
+
+    /// Initialize context
+    result = ma_context_init(nullptr, 0, nullptr, this->context.get());
+    if (result != MA_SUCCESS)
+    {
+        qCWarning(chatterinoSound) << "Error initializing context:" << result;
+        return;
+    }
+
+    /// Initialize resource manager
+    auto resourceManagerConfig = ma_resource_manager_config_init();
+    resourceManagerConfig.decodedFormat = ma_format_f32;
+    // Use native channel count
+    resourceManagerConfig.decodedChannels = 0;
+    resourceManagerConfig.decodedSampleRate = 48000;
+
+    result = ma_resource_manager_init(&resourceManagerConfig,
+                                      this->resourceManager.get());
+    if (result != MA_SUCCESS)
+    {
+        qCWarning(chatterinoSound)
+            << "Error initializing resource manager:" << result;
+        return;
+    }
+
+    /// Load default sound
+    QFile defaultPingFile(":/sounds/ping2.wav");
+    if (!defaultPingFile.open(QIODevice::ReadOnly))
+    {
+        qCWarning(chatterinoSound) << "Error loading default ping sound";
+        return;
+    }
+    this->defaultPingData = defaultPingFile.readAll();
+
+    /// Initialize a sound device
+    auto deviceConfig = ma_device_config_init(ma_device_type_playback);
+    deviceConfig.playback.pDeviceID = nullptr;
+    deviceConfig.playback.format = this->resourceManager->config.decodedFormat;
+    deviceConfig.playback.channels = 0;
+    deviceConfig.pulse.pStreamNamePlayback = "Chatterino MA";
+    deviceConfig.sampleRate = this->resourceManager->config.decodedSampleRate;
+    deviceConfig.dataCallback = ma_engine_data_callback_internal;
+    deviceConfig.pUserData = this->engine.get();
+
+    result =
+        ma_device_init(this->context.get(), &deviceConfig, this->device.get());
+    if (result != MA_SUCCESS)
+    {
+        qCWarning(chatterinoSound) << "Error initializing device:" << result;
+        return;
+    }
+
+    result = ma_device_start(this->device.get());
+    if (result != MA_SUCCESS)
+    {
+        qCWarning(chatterinoSound) << "Error starting device:" << result;
+        return;
+    }
+
+    /// Initialize engine
+    auto engineConfig = ma_engine_config_init();
+    engineConfig.pResourceManager = this->resourceManager.get();
+    engineConfig.pDevice = this->device.get();
+    engineConfig.pContext = this->context.get();
+
+    result = ma_engine_init(&engineConfig, this->engine.get());
+    if (result != MA_SUCCESS)
+    {
+        qCWarning(chatterinoSound) << "Error initializing engine:" << result;
+        return;
+    }
+
+    /// Initialize default ping sounds
+    {
+        // TODO: Can we optimize this?
+        BenchmarkGuard b("init sounds");
+
+        ma_uint32 soundFlags = 0;
+        // Decode the sound during loading instead of during playback
+        soundFlags |= MA_SOUND_FLAG_DECODE;
+        // Disable pitch control (we don't use it, so this saves some performance)
+        soundFlags |= MA_SOUND_FLAG_NO_PITCH;
+        // Disable spatialization control, this brings the volume up to "normal levels"
+        soundFlags |= MA_SOUND_FLAG_NO_SPATIALIZATION;
+
+        auto decoderConfig = ma_decoder_config_init(ma_format_f32, 0, 48000);
+        // This must match the encoding format of our default ping sound
+        decoderConfig.encodingFormat = ma_encoding_format_wav;
+
+        for (auto i = 0; i < NUM_SOUNDS; ++i)
+        {
+            auto dec = std::make_unique<ma_decoder>();
+            auto snd = std::make_unique<ma_sound>();
+
+            result = ma_decoder_init_memory(
+                (void *)this->defaultPingData.data(),
+                this->defaultPingData.size() * sizeof(char), &decoderConfig,
+                dec.get());
+            if (result != MA_SUCCESS)
+            {
+                qCWarning(chatterinoSound)
+                    << "Error initializing default ping decoder from memory:"
+                    << result;
+                return;
+            }
+
+            result = ma_sound_init_from_data_source(
+                this->engine.get(), dec.get(), soundFlags, nullptr, snd.get());
+            if (result != MA_SUCCESS)
+            {
+                qCWarning(chatterinoSound)
+                    << "Error initializing default sound from data source:"
+                    << result;
+                return;
+            }
+
+            this->defaultPingDecoders.emplace_back(std::move(dec));
+            this->defaultPingSounds.emplace_back(std::move(snd));
+        }
+    }
+
+    qCInfo(chatterinoSound) << "miniaudio sound system initialized";
+
+    this->initialized = true;
+}
+
+SoundController::~SoundController()
+{
+    // NOTE: This destructor is never called because the `runGui` function calls _exit before that happens
+    // I have manually called the destructor prior to _exit being called to ensure this logic is sound
+
+    for (const auto &snd : this->defaultPingSounds)
+    {
+        ma_sound_uninit(snd.get());
+    }
+    for (const auto &dec : this->defaultPingDecoders)
+    {
+        ma_decoder_uninit(dec.get());
+    }
+
+    ma_engine_uninit(this->engine.get());
+    ma_device_uninit(this->device.get());
+    ma_resource_manager_uninit(this->resourceManager.get());
+    ma_context_uninit(this->context.get());
+}
+
+void SoundController::play(const QUrl &sound)
+{
+    static size_t i = 0;
+
+    this->tgPlay.guard();
+
+    if (!this->initialized)
+    {
+        qCWarning(chatterinoSound) << "Can't play sound, sound controller "
+                                      "didn't initialize correctly";
+        return;
+    }
+
+    if (sound.isLocalFile())
+    {
+        auto soundPath = sound.toLocalFile();
+        auto result = ma_engine_play_sound(this->engine.get(),
+                                           qPrintable(soundPath), nullptr);
+        if (result != MA_SUCCESS)
+        {
+            qCWarning(chatterinoSound) << "Failed to play sound" << sound
+                                       << soundPath << ":" << result;
+        }
+
+        return;
+    }
+
+    // Play default sound, loaded from our resources in the constructor
+    auto &snd = this->defaultPingSounds[++i % NUM_SOUNDS];
+    ma_sound_seek_to_pcm_frame(snd.get(), 0);
+    auto result = ma_sound_start(snd.get());
+    if (result != MA_SUCCESS)
+    {
+        qCWarning(chatterinoSound) << "Failed to play default ping" << result;
+    }
+}
+
+}  // namespace chatterino

src/controllers/sound/SoundController.hpp

@@ -0,0 +1,70 @@
+#pragma once
+
+#include "common/Singleton.hpp"
+#include "util/ThreadGuard.hpp"
+
+#include <QByteArray>
+#include <QString>
+#include <QUrl>
+
+#include <memory>
+#include <vector>
+
+struct ma_engine;
+struct ma_device;
+struct ma_resource_manager;
+struct ma_context;
+struct ma_sound;
+struct ma_decoder;
+
+namespace chatterino {
+
+class Settings;
+class Paths;
+
+/**
+ * @brief Handles sound loading & playback
+ **/
+class SoundController : public Singleton
+{
+    SoundController();
+
+    void initialize(Settings &settings, Paths &paths) override;
+
+public:
+    ~SoundController() override;
+
+    // Play a sound from the given url
+    // If the url points to something that isn't a local file, it will play
+    // the default sound initialized in the initialize method
+    void play(const QUrl &sound);
+
+private:
+    // Used for selecting & initializing an appropriate sound backend
+    std::unique_ptr<ma_context> context;
+    // Used for storing & reusing sounds to be played
+    std::unique_ptr<ma_resource_manager> resourceManager;
+    // The sound device we're playing sound into
+    std::unique_ptr<ma_device> device;
+    // The engine is a high-level API for playing sounds from paths in a simple & efficient-enough manner
+    std::unique_ptr<ma_engine> engine;
+
+    // Stores the data of our default ping sounds
+    QByteArray defaultPingData;
+    // Stores N decoders for simultaneous default ping playback.
+    // We can't use the engine API for this as this requires direct access to a custom data_source
+    std::vector<std::unique_ptr<ma_decoder>> defaultPingDecoders;
+    // Stores N sounds for simultaneous default ping playback
+    // We can't use the engine API for this as this requires direct access to a custom data_source
+    std::vector<std::unique_ptr<ma_sound>> defaultPingSounds;
+
+    // Thread guard for the play method
+    // Ensures play is only ever called from the same thread
+    ThreadGuard tgPlay;
+
+    bool initialized{false};
+
+    friend class Application;
+};
+
+}  // namespace chatterino

src/messages/SharedMessageBuilder.cpp

@@ -6,6 +6,7 @@
 #include "controllers/ignores/IgnoreController.hpp"
 #include "controllers/ignores/IgnorePhrase.hpp"
 #include "controllers/nicknames/Nickname.hpp"
+#include "controllers/sound/SoundController.hpp"
 #include "messages/Message.hpp"
 #include "messages/MessageElement.hpp"
 #include "providers/twitch/TwitchBadge.hpp"
@@ -16,34 +17,33 @@
 #include "util/StreamerMode.hpp"
 
 #include <QFileInfo>
-#include <QMediaPlayer>
-
-namespace chatterino {
 
 namespace {
 
-    /**
+using namespace chatterino;
+
+/**
  * Gets the default sound url if the user set one,
  * or the chatterino default ping sound if no url is set.
  */
-    QUrl getFallbackHighlightSound()
-    {
+QUrl getFallbackHighlightSound()
+{
     QString path = getSettings()->pathHighlightSound;
-        bool fileExists = !path.isEmpty() && QFileInfo::exists(path) &&
-                          QFileInfo(path).isFile();
+    bool fileExists =
+        !path.isEmpty() && QFileInfo::exists(path) && QFileInfo(path).isFile();
 
     if (fileExists)
     {
         return QUrl::fromLocalFile(path);
     }
-        else
-        {
+
     return QUrl("qrc:/sounds/ping2.wav");
-        }
-    }
+}
 
 }  // namespace
 
+namespace chatterino {
+
 SharedMessageBuilder::SharedMessageBuilder(
     Channel *_channel, const Communi::IrcPrivateMessage *_ircMessage,
     const MessageParseArgs &_args)
@@ -198,23 +198,8 @@ void SharedMessageBuilder::appendChannelName()
         ->setLink(link);
 }
 
-inline QMediaPlayer *getPlayer()
-{
-    if (isGuiThread())
-    {
-        static auto player = new QMediaPlayer;
-        return player;
-    }
-    else
-    {
-        return nullptr;
-    }
-}
-
 void SharedMessageBuilder::triggerHighlights()
 {
-    static QUrl currentPlayerUrl;
-
     if (isInStreamerMode() && getSettings()->streamerModeMuteMentions)
     {
         // We are in streamer mode with muting mention sounds enabled. Do nothing.
@@ -232,19 +217,7 @@ void SharedMessageBuilder::triggerHighlights()
 
     if (this->highlightSound_ && resolveFocus)
     {
-        if (auto player = getPlayer())
-        {
-            // Set media if the highlight sound url has changed, or if media is buffered
-            if (currentPlayerUrl != this->highlightSoundUrl_ ||
-                player->mediaStatus() == QMediaPlayer::BufferedMedia)
-            {
-                player->setMedia(this->highlightSoundUrl_);
-
-                currentPlayerUrl = this->highlightSoundUrl_;
-            }
-
-            player->play();
-        }
+        getApp()->sound->play(this->highlightSoundUrl_);
     }
 
     if (this->highlightAlert_)

src/providers/twitch/TwitchMessageBuilder.cpp

@@ -35,7 +35,6 @@
 #include <boost/variant.hpp>
 #include <QColor>
 #include <QDebug>
-#include <QMediaPlayer>
 #include <QStringRef>
 
 namespace {

src/widgets/settingspages/AboutPage.cpp

@@ -110,6 +110,9 @@ AboutPage::AboutPage()
             addLicense(form.getElement(), "semver",
                        "https://github.com/Neargye/semver",
                        ":/licenses/semver.txt");
+            addLicense(form.getElement(), "miniaudio",
+                       "https://github.com/mackron/miniaudio",
+                       ":/licenses/miniaudio.txt");
         }
 
         // Attributions