Make tests more platform agnostic (#4650)

Use QTemporaryDir to create the test directory
Add option to use httpbin over local docker
Increase delay for opening a connection

Co-authored-by: Rasmus Karlsson <rasmus.karlsson@pajlada.com>

benchmarks/src/Highlights.cpp

@@ -12,6 +12,7 @@
 #include <benchmark/benchmark.h>
 #include <QDebug>
 #include <QString>
+#include <QTemporaryDir>
 
 using namespace chatterino;
 
@@ -66,7 +67,8 @@ public:
 static void BM_HighlightTest(benchmark::State &state)
 {
     MockApplication mockApplication;
-    Settings settings("/tmp/c2-mock");
+    QTemporaryDir settingsDir;
+    Settings settings(settingsDir.path());
 
     std::string message =
         R"(@badge-info=subscriber/34;badges=moderator/1,subscriber/24;color=#FF0000;display-name=테스트계정420;emotes=41:6-13,15-22;flags=;id=a3196c7e-be4c-4b49-9c5a-8b8302b50c2a;mod=1;room-id=11148817;subscriber=1;tmi-sent-ts=1590922213730;turbo=0;user-id=117166826;user-type=mod :testaccount_420!testaccount_420@testaccount_420.tmi.twitch.tv PRIVMSG #pajlada :-tags Kreygasm,Kreygasm (no space))";

benchmarks/src/main.cpp

@@ -3,6 +3,7 @@
 #include <benchmark/benchmark.h>
 #include <QApplication>
 #include <QtConcurrent>
+#include <QTemporaryDir>
 
 using namespace chatterino;
 
@@ -12,12 +13,15 @@ int main(int argc, char **argv)
 
     ::benchmark::Initialize(&argc, argv);
 
-    // Ensure settings are initialized before any tests are run
-    chatterino::Settings settings("/tmp/c2-empty-mock");
+    // Ensure settings are initialized before any benchmarks are run
+    QTemporaryDir settingsDir;
+    settingsDir.setAutoRemove(false);  // we'll remove it manually
+    chatterino::Settings settings(settingsDir.path());
 
-    QtConcurrent::run([&app] {
+    QtConcurrent::run([&app, &settingsDir]() mutable {
         ::benchmark::RunSpecifiedBenchmarks();
 
+        settingsDir.remove();
         app.exit(0);
     });
 

tests/CMakeLists.txt

@@ -1,5 +1,7 @@
 project(chatterino-test)
 
+option(CHATTERINO_TEST_USE_PUBLIC_HTTPBIN "Use public httpbin for testing network requests" OFF)
+
 set(test_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/src/main.cpp
     ${CMAKE_CURRENT_LIST_DIR}/src/ChannelChatters.cpp
@@ -54,6 +56,10 @@ if(CHATTERINO_ENABLE_LTO)
         PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
 endif()
 
+if(CHATTERINO_TEST_USE_PUBLIC_HTTPBIN)
+    target_compile_definitions(${PROJECT_NAME} PRIVATE CHATTERINO_TEST_USE_PUBLIC_HTTPBIN)
+endif()
+
 # gtest_add_tests manages to discover the tests because it looks through the source files
 # HOWEVER, it fails to run, because we have some bug that causes the QApplication exit to stall when no network requests have been made.
 # ctest runs each test individually, so for now we require that testers just run the ./bin/chatterino-test binary without any filters applied

tests/src/HighlightController.cpp

@@ -19,6 +19,7 @@
 #include <QDir>
 #include <QFile>
 #include <QString>
+#include <QTemporaryDir>
 
 using namespace chatterino;
 using ::testing::Exactly;
@@ -178,9 +179,9 @@ protected:
     void SetUp() override
     {
         // Write default settings to the mock settings json file
-        ASSERT_TRUE(QDir().mkpath("/tmp/c2-tests"));
+        this->settingsDir_ = std::make_unique<QTemporaryDir>();
 
-        QFile settingsFile("/tmp/c2-tests/settings.json");
+        QFile settingsFile(this->settingsDir_->filePath("settings.json"));
         ASSERT_TRUE(settingsFile.open(QIODevice::WriteOnly | QIODevice::Text));
         ASSERT_GT(settingsFile.write(DEFAULT_SETTINGS.toUtf8()), 0);
         ASSERT_TRUE(settingsFile.flush());
@@ -194,7 +195,7 @@ protected:
         EXPECT_CALL(*this->mockHelix, update).Times(Exactly(1));
 
         this->mockApplication = std::make_unique<MockApplication>();
-        this->settings = std::make_unique<Settings>("/tmp/c2-tests");
+        this->settings = std::make_unique<Settings>(this->settingsDir_->path());
         this->paths = std::make_unique<Paths>();
 
         this->controller = std::make_unique<HighlightController>();
@@ -206,16 +207,19 @@ protected:
 
     void TearDown() override
     {
-        ASSERT_TRUE(QDir("/tmp/c2-tests").removeRecursively());
         this->mockApplication.reset();
         this->settings.reset();
         this->paths.reset();
 
         this->controller.reset();
 
+        this->settingsDir_.reset();
+
         delete this->mockHelix;
     }
 
+    std::unique_ptr<QTemporaryDir> settingsDir_;
+
     std::unique_ptr<MockApplication> mockApplication;
     std::unique_ptr<Settings> settings;
     std::unique_ptr<Paths> paths;

tests/src/InputCompletion.cpp

@@ -18,6 +18,7 @@
 #include <QFile>
 #include <QModelIndex>
 #include <QString>
+#include <QTemporaryDir>
 
 namespace {
 
@@ -122,9 +123,9 @@ protected:
     void SetUp() override
     {
         // Write default settings to the mock settings json file
-        ASSERT_TRUE(QDir().mkpath("/tmp/c2-tests"));
+        this->settingsDir_ = std::make_unique<QTemporaryDir>();
 
-        QFile settingsFile("/tmp/c2-tests/settings.json");
+        QFile settingsFile(this->settingsDir_->filePath("settings.json"));
         ASSERT_TRUE(settingsFile.open(QIODevice::WriteOnly | QIODevice::Text));
         ASSERT_GT(settingsFile.write(DEFAULT_SETTINGS.toUtf8()), 0);
         ASSERT_TRUE(settingsFile.flush());
@@ -137,7 +138,7 @@ protected:
         EXPECT_CALL(*this->mockHelix, update).Times(Exactly(1));
 
         this->mockApplication = std::make_unique<MockApplication>();
-        this->settings = std::make_unique<Settings>("/tmp/c2-tests");
+        this->settings = std::make_unique<Settings>(this->settingsDir_->path());
         this->paths = std::make_unique<Paths>();
 
         this->mockApplication->accounts.initialize(*this->settings,
@@ -153,15 +154,18 @@ protected:
 
     void TearDown() override
     {
-        ASSERT_TRUE(QDir("/tmp/c2-tests").removeRecursively());
         this->mockApplication.reset();
         this->settings.reset();
         this->paths.reset();
         this->mockHelix.reset();
         this->completionModel.reset();
         this->channelPtr.reset();
+
+        this->settingsDir_.reset();
     }
 
+    std::unique_ptr<QTemporaryDir> settingsDir_;
+
     std::unique_ptr<MockApplication> mockApplication;
     std::unique_ptr<Settings> settings;
     std::unique_ptr<Paths> paths;
@@ -222,8 +226,14 @@ protected:
 
 TEST_F(InputCompletionTest, EmoteNameFiltering)
 {
+    // The completion doesn't guarantee an ordering for a specific category of emotes.
+    // This tests a specific implementation of the underlying std::unordered_map,
+    // so depending on the standard library used when compiling, this might yield
+    // different results.
+
     auto completion = queryEmoteCompletion(":feels");
     ASSERT_EQ(completion.size(), 3);
+    // all these matches are BTTV global emotes
     ASSERT_EQ(completion[0].displayName, "FeelsBirthdayMan");
     ASSERT_EQ(completion[1].displayName, "FeelsBadMan");
     ASSERT_EQ(completion[2].displayName, "FeelsGoodMan");
@@ -231,6 +241,7 @@ TEST_F(InputCompletionTest, EmoteNameFiltering)
     completion = queryEmoteCompletion(":)");
     ASSERT_EQ(completion.size(), 3);
     ASSERT_EQ(completion[0].displayName, ":)");  // Exact match with : prefix
+    // all these matches are Twitch global emotes
     ASSERT_EQ(completion[1].displayName, ":-)");
     ASSERT_EQ(completion[2].displayName, "B-)");
 

tests/src/NetworkRequest.cpp

@@ -12,8 +12,11 @@ using namespace chatterino;
 
 namespace {
 
-// Change to http://httpbin.org if you don't want to run the docker image yourself to test this
+#ifdef CHATTERINO_TEST_USE_PUBLIC_HTTPBIN
+const char *const HTTPBIN_BASE_URL = "http://httpbin.org";
+#else
 const char *const HTTPBIN_BASE_URL = "http://127.0.0.1:9051";
+#endif
 
 QString getStatusURL(int code)
 {

tests/src/TwitchPubSubClient.cpp

@@ -48,7 +48,7 @@ TEST(TwitchPubSubClient, ServerRespondsToPings)
 
     pubSub->listenToTopic("test");
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 1);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);
@@ -211,7 +211,7 @@ TEST(TwitchPubSubClient, ExceedTopicLimitSingleStep)
         pubSub->listenToTopic("test");
     }
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 2);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);
@@ -242,7 +242,7 @@ TEST(TwitchPubSubClient, ReceivedWhisper)
 
     pubSub->listenToTopic("whispers.123456");
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 1);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);
@@ -324,7 +324,7 @@ TEST(TwitchPubSubClient, MissingToken)
 
     pubSub->listenToTopic("chat_moderator_actions.123456.123456");
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 1);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);

tests/src/main.cpp

@@ -1,14 +1,10 @@
 #include "common/NetworkManager.hpp"
-#include "common/NetworkRequest.hpp"
-#include "common/NetworkResult.hpp"
-#include "common/Outcome.hpp"
-#include "common/QLogging.hpp"
-#include "providers/twitch/api/Helix.hpp"
 #include "singletons/Settings.hpp"
 
 #include <gtest/gtest.h>
 #include <QApplication>
 #include <QJsonArray>
+#include <QLoggingCategory>
 #include <QtConcurrent>
 #include <QTimer>
 
@@ -25,17 +21,23 @@ int main(int argc, char **argv)
 
 #ifdef SUPPORT_QT_NETWORK_TESTS
     QApplication app(argc, argv);
+    // make sure to always debug-log
+    QLoggingCategory::setFilterRules("*.debug=true");
 
     chatterino::NetworkManager::init();
 
     // Ensure settings are initialized before any tests are run
-    chatterino::Settings settings("/tmp/c2-empty-test");
+    QTemporaryDir settingsDir;
+    settingsDir.setAutoRemove(false);  // we'll remove it manually
+    qDebug() << "Settings directory:" << settingsDir.path();
+    chatterino::Settings settings(settingsDir.path());
 
-    QtConcurrent::run([&app] {
+    QtConcurrent::run([&app, &settingsDir]() mutable {
         auto res = RUN_ALL_TESTS();
 
         chatterino::NetworkManager::deinit();
 
+        settingsDir.remove();
         app.exit(res);
     });