#include "messages/Image.hpp" #include "Application.hpp" #include "common/Common.hpp" #include "common/NetworkRequest.hpp" #include "common/NetworkResult.hpp" #include "common/Outcome.hpp" #include "common/QLogging.hpp" #include "debug/AssertInGuiThread.hpp" #include "debug/Benchmark.hpp" #include "singletons/Emotes.hpp" #include "singletons/helper/GifTimer.hpp" #include "singletons/WindowManager.hpp" #include "util/DebugCount.hpp" #include "util/PostToThread.hpp" #include #include #include #include #include #include #include #include #include #include // Duration between each check of every Image instance const auto IMAGE_POOL_CLEANUP_INTERVAL = std::chrono::minutes(1); // Duration since last usage of Image pixmap before expiration of frames const auto IMAGE_POOL_IMAGE_LIFETIME = std::chrono::minutes(10); namespace chatterino { namespace detail { // Frames Frames::Frames() { DebugCount::increase("images"); } Frames::Frames(QVector> &&frames) : items_(std::move(frames)) { assertInGuiThread(); DebugCount::increase("images"); if (!this->empty()) { DebugCount::increase("loaded images"); } if (this->animated()) { DebugCount::increase("animated images"); this->gifTimerConnection_ = getApp()->emotes->gifTimer.signal.connect([this] { this->advance(); }); } auto totalLength = std::accumulate(this->items_.begin(), this->items_.end(), 0UL, [](auto init, auto &&frame) { return init + frame.duration; }); if (totalLength == 0) { this->durationOffset_ = 0; } else { this->durationOffset_ = std::min( int(getApp()->emotes->gifTimer.position() % totalLength), 60000); } this->processOffset(); DebugCount::increase("image bytes", this->memoryUsage()); DebugCount::increase("image bytes (ever loaded)", this->memoryUsage()); } Frames::~Frames() { assertInGuiThread(); DebugCount::decrease("images"); if (!this->empty()) { DebugCount::decrease("loaded images"); } if (this->animated()) { DebugCount::decrease("animated images"); } DebugCount::decrease("image bytes", this->memoryUsage()); DebugCount::increase("image bytes (ever unloaded)", this->memoryUsage()); this->gifTimerConnection_.disconnect(); } int64_t Frames::memoryUsage() const { int64_t usage = 0; for (const auto &frame : this->items_) { auto sz = frame.image.size(); auto area = sz.width() * sz.height(); auto memory = area * frame.image.depth(); usage += memory; } return usage; } void Frames::advance() { this->durationOffset_ += GIF_FRAME_LENGTH; this->processOffset(); } void Frames::processOffset() { if (this->items_.isEmpty()) { return; } while (true) { this->index_ %= this->items_.size(); if (this->durationOffset_ > this->items_[this->index_].duration) { this->durationOffset_ -= this->items_[this->index_].duration; this->index_ = (this->index_ + 1) % this->items_.size(); } else { break; } } } void Frames::clear() { assertInGuiThread(); if (!this->empty()) { DebugCount::decrease("loaded images"); } DebugCount::decrease("image bytes", this->memoryUsage()); DebugCount::increase("image bytes (ever unloaded)", this->memoryUsage()); this->items_.clear(); this->index_ = 0; this->durationOffset_ = 0; this->gifTimerConnection_.disconnect(); } bool Frames::empty() const { return this->items_.empty(); } bool Frames::animated() const { return this->items_.size() > 1; } boost::optional Frames::current() const { if (this->items_.size() == 0) return boost::none; return this->items_[this->index_].image; } boost::optional Frames::first() const { if (this->items_.size() == 0) return boost::none; return this->items_.front().image; } // functions QVector> readFrames(QImageReader &reader, const Url &url) { QVector> frames; frames.reserve(reader.imageCount()); QImage image; for (int index = 0; index < reader.imageCount(); ++index) { if (reader.read(&image)) { // It seems that browsers have special logic for fast animations. // This implements Chrome and Firefox's behavior which uses // a duration of 100 ms for any frames that specify a duration of <= 10 ms. // See http://webkit.org/b/36082 for more information. // https://github.com/SevenTV/chatterino7/issues/46#issuecomment-1010595231 int duration = reader.nextImageDelay(); if (duration <= 10) duration = 100; duration = std::max(20, duration); frames.push_back(Frame{std::move(image), duration}); } } if (frames.size() == 0) { qCDebug(chatterinoImage) << "Error while reading image" << url.string << ": '" << reader.errorString() << "'"; } return frames; } // parsed template void assignDelayed( std::queue>>> &queued, std::mutex &mutex, std::atomic_bool &loadedEventQueued) { std::lock_guard lock(mutex); int i = 0; while (!queued.empty()) { auto front = std::move(queued.front()); queued.pop(); // Call Assign with the vector of frames front.first(std::move(front.second)); if (++i > 50) { QTimer::singleShot(3, [&] { assignDelayed(queued, mutex, loadedEventQueued); }); return; } } getApp()->windows->forceLayoutChannelViews(); loadedEventQueued = false; } template auto makeConvertCallback(const QVector> &parsed, Assign assign) { static std::queue>>> queued; static std::mutex mutex; static std::atomic_bool loadedEventQueued{false}; return [parsed, assign] { // convert to pixmap QVector> frames; frames.reserve(parsed.size()); std::transform(parsed.begin(), parsed.end(), std::back_inserter(frames), [](auto &frame) { return Frame{ QPixmap::fromImage(frame.image), frame.duration}; }); // put into stack std::lock_guard lock(mutex); queued.emplace(assign, frames); if (!loadedEventQueued) { loadedEventQueued = true; QTimer::singleShot(100, [=] { assignDelayed(queued, mutex, loadedEventQueued); }); } }; } } // namespace detail // IMAGE2 Image::~Image() { #ifndef DISABLE_IMAGE_EXPIRATION_POOL ImageExpirationPool::instance().removeImagePtr(this); #endif if (this->empty_ && !this->frames_) { // No data in this image, don't bother trying to release it // The reason we do this check is that we keep a few (or one) static empty image around that are deconstructed at the end of the programs lifecycle, and we want to prevent the isGuiThread call to be called after the QApplication has been exited return; } // Ensure the destructor for our frames is called in the GUI thread // If the Image destructor is called outside of the GUI thread, move the // ownership of the frames to the GUI thread, otherwise the frames will be // destructed as part as we go out of scope if (!isGuiThread()) { postToThread([frames = this->frames_.release()]() { delete frames; }); } } ImagePtr Image::fromUrl(const Url &url, qreal scale) { static std::unordered_map> cache; static std::mutex mutex; std::lock_guard lock(mutex); auto shared = cache[url].lock(); if (!shared) { cache[url] = shared = ImagePtr(new Image(url, scale)); } return shared; } ImagePtr Image::fromResourcePixmap(const QPixmap &pixmap, qreal scale) { using key_t = std::pair; static std::unordered_map, boost::hash> cache; static std::mutex mutex; std::lock_guard lock(mutex); auto it = cache.find({&pixmap, scale}); if (it != cache.end()) { auto shared = it->second.lock(); if (shared) { return shared; } else { cache.erase(it); } } auto newImage = ImagePtr(new Image(scale)); newImage->setPixmap(pixmap); // store in cache cache.insert({{&pixmap, scale}, std::weak_ptr(newImage)}); return newImage; } ImagePtr Image::getEmpty() { static auto empty = ImagePtr(new Image); return empty; } ImagePtr getEmptyImagePtr() { return Image::getEmpty(); } Image::Image() : empty_(true) { } Image::Image(const Url &url, qreal scale) : url_(url) , scale_(scale) , shouldLoad_(true) , frames_(std::make_unique()) { } Image::Image(qreal scale) : scale_(scale) , frames_(std::make_unique()) { } void Image::setPixmap(const QPixmap &pixmap) { auto setFrames = [shared = this->shared_from_this(), pixmap]() { shared->frames_ = std::make_unique( QVector>{detail::Frame{pixmap, 1}}); }; if (isGuiThread()) { setFrames(); } else { postToThread(setFrames); } } const Url &Image::url() const { return this->url_; } bool Image::loaded() const { assertInGuiThread(); return bool(this->frames_->current()); } boost::optional Image::pixmapOrLoad() const { assertInGuiThread(); // Mark the image as just used. // Any time this Image is painted, this method is invoked. // See src/messages/layouts/MessageLayoutElement.cpp ImageLayoutElement::paint, for example. this->lastUsed_ = std::chrono::steady_clock::now(); this->load(); return this->frames_->current(); } void Image::load() const { assertInGuiThread(); if (this->shouldLoad_) { Image *this2 = const_cast(this); this2->shouldLoad_ = false; this2->actuallyLoad(); #ifndef DISABLE_IMAGE_EXPIRATION_POOL ImageExpirationPool::instance().addImagePtr(this2->shared_from_this()); #endif } } qreal Image::scale() const { return this->scale_; } bool Image::isEmpty() const { return this->empty_; } bool Image::animated() const { assertInGuiThread(); return this->frames_->animated(); } int Image::width() const { assertInGuiThread(); if (auto pixmap = this->frames_->first()) return int(pixmap->width() * this->scale_); else return 16; } int Image::height() const { assertInGuiThread(); if (auto pixmap = this->frames_->first()) return int(pixmap->height() * this->scale_); else return 16; } void Image::actuallyLoad() { NetworkRequest(this->url().string) .concurrent() .cache() .onSuccess([weak = weakOf(this)](auto result) -> Outcome { auto shared = weak.lock(); if (!shared) return Failure; auto data = result.getData(); // const cast since we are only reading from it QBuffer buffer(const_cast(&data)); buffer.open(QIODevice::ReadOnly); QImageReader reader(&buffer); if (!reader.canRead()) { qCDebug(chatterinoImage) << "Error: image cant be read " << shared->url().string; shared->empty_ = true; return Failure; } const auto size = reader.size(); if (size.isEmpty()) { shared->empty_ = true; return Failure; } // returns 1 for non-animated formats if (reader.imageCount() <= 0) { qCDebug(chatterinoImage) << "Error: image has less than 1 frame " << shared->url().string << ": " << reader.errorString(); shared->empty_ = true; return Failure; } // use "double" to prevent int overflows if (double(size.width()) * double(size.height()) * double(reader.imageCount()) * 4.0 > double(Image::maxBytesRam)) { qCDebug(chatterinoImage) << "image too large in RAM"; shared->empty_ = true; return Failure; } auto parsed = detail::readFrames(reader, shared->url()); postToThread(makeConvertCallback(parsed, [weak](auto &&frames) { if (auto shared = weak.lock()) { shared->frames_ = std::make_unique(std::move(frames)); } })); return Success; }) .onError([weak = weakOf(this)](auto /*result*/) { auto shared = weak.lock(); if (!shared) return false; // fourtf: is this the right thing to do? shared->empty_ = true; return true; }) .execute(); } void Image::expireFrames() { assertInGuiThread(); this->frames_->clear(); this->shouldLoad_ = true; // Mark as needing load again } #ifndef DISABLE_IMAGE_EXPIRATION_POOL ImageExpirationPool::ImageExpirationPool() : freeTimer_(new QTimer) { QObject::connect(this->freeTimer_, &QTimer::timeout, [this] { if (isGuiThread()) { this->freeOld(); } else { postToThread([this] { this->freeOld(); }); } }); this->freeTimer_->start( std::chrono::duration_cast( IMAGE_POOL_CLEANUP_INTERVAL)); } ImageExpirationPool &ImageExpirationPool::instance() { static auto *instance = new ImageExpirationPool; return *instance; } void ImageExpirationPool::addImagePtr(ImagePtr imgPtr) { std::lock_guard lock(this->mutex_); this->allImages_.emplace(imgPtr.get(), std::weak_ptr(imgPtr)); } void ImageExpirationPool::removeImagePtr(Image *rawPtr) { std::lock_guard lock(this->mutex_); this->allImages_.erase(rawPtr); } void ImageExpirationPool::freeAll() { { std::lock_guard lock(this->mutex_); for (auto it = this->allImages_.begin(); it != this->allImages_.end();) { auto img = it->second.lock(); img->expireFrames(); it = this->allImages_.erase(it); } } this->freeOld(); } void ImageExpirationPool::freeOld() { std::lock_guard lock(this->mutex_); size_t numExpired = 0; size_t eligible = 0; auto now = std::chrono::steady_clock::now(); for (auto it = this->allImages_.begin(); it != this->allImages_.end();) { auto img = it->second.lock(); if (!img) { // This can only really happen from a race condition because ~Image // should remove itself from the ImageExpirationPool automatically. it = this->allImages_.erase(it); continue; } if (img->frames_->empty()) { // No frame data, nothing to do ++it; continue; } ++eligible; // Check if image has expired and, if so, expire its frame data auto diff = now - img->lastUsed_; if (diff > IMAGE_POOL_IMAGE_LIFETIME) { ++numExpired; img->expireFrames(); // erase without mutex locking issue it = this->allImages_.erase(it); continue; } ++it; } # ifndef NDEBUG qCDebug(chatterinoImage) << "freed frame data for" << numExpired << "/" << eligible << "eligible images"; # endif DebugCount::set("last image gc: expired", numExpired); DebugCount::set("last image gc: eligible", eligible); DebugCount::set("last image gc: left after gc", this->allImages_.size()); } #endif } // namespace chatterino