fix: waylib: back off failed Vulkan render imports under load

Avoid treating failed Vulkan render attempts as completed frames when the
Vulkan+GL bridge is under pressure. If acquiring the output render target
fails, skip the current frame without committing or clearing the dirty state.
If output commit fails after rendering a buffer, mark the output dirty again
and retry with a short frame-based backoff.

Also validate GL texture creation for Vulkan dmabuf imports and readback
fallbacks before handing them to Qt RHI, preventing invalid texture IDs from
reaching QQuickRenderTarget. Surface texture updates now keep the previous
texture and briefly back off Vulkan-only retries when imports fail, avoiding
tight retry loops during heavy CPU/GPU pressure.

The GLES2 path is still left unchanged.

Author: LFRon

waylib/src/server/qtquick/woutputrenderwindow.cpp

@@ -204,6 +204,42 @@ class Q_DECL_HIDDEN OutputHelper : public WOutputHelper
         return output()->output()->handle();
     }
 
+#ifdef ENABLE_VULKAN_RENDER
+    inline bool isVulkanRenderer() const {
+        return m_output && m_output->output() && m_output->output()->renderer()
+            && wlr_renderer_is_vk(m_output->output()->renderer()->handle());
+    }
+
+    inline bool shouldDeferVulkanRenderRetry() {
+        if (!isVulkanRenderer() || m_vulkanRenderRetryDelayFrames <= 0)
+            return false;
+
+        --m_vulkanRenderRetryDelayFrames;
+        scheduleFrame();
+        return true;
+    }
+
+    inline bool noteVulkanRenderFailed() {
+        if (!isVulkanRenderer())
+            return false;
+
+        m_vulkanRenderFailureBackoffFrames = m_vulkanRenderFailureBackoffFrames
+            ? qMin(m_vulkanRenderFailureBackoffFrames * 2, 4)
+            : 1;
+        m_vulkanRenderRetryDelayFrames = m_vulkanRenderFailureBackoffFrames;
+        scheduleFrame();
+        return true;
+    }
+
+    inline void noteVulkanRenderSucceeded() {
+        if (!isVulkanRenderer())
+            return;
+
+        m_vulkanRenderFailureBackoffFrames = 0;
+        m_vulkanRenderRetryDelayFrames = 0;
+    }
+#endif
+
     inline WOutputRenderWindow *renderWindow() const {
         return static_cast<WOutputRenderWindow*>(parent());
     }
@@ -273,6 +309,10 @@ class Q_DECL_HIDDEN OutputHelper : public WOutputHelper
     BufferRendererProxy *m_cursorLayerProxy = nullptr;
     bool m_cursorDirty = false;
     bool m_hardwareCursorRenderComplete = false;
+#ifdef ENABLE_VULKAN_RENDER
+    int m_vulkanRenderFailureBackoffFrames = 0;
+    int m_vulkanRenderRetryDelayFrames = 0;
+#endif
 
     // for compositeLayers
     QPointer<WOutputViewport> m_output2;
@@ -1548,9 +1588,25 @@ WOutputRenderWindowPrivate::doRenderOutputs(qw_output *needsFrameOutput, const Q
         if (!helper->output()->depends().isEmpty())
             updateDirtyNodes();
 
+#ifdef ENABLE_VULKAN_RENDER
+        if (helper->shouldDeferVulkanRenderRetry())
+            continue;
+#endif
+
         qw_buffer *buffer = helper->beginRender(helper->bufferRenderer(), helper->output()->output()->size(), format,
                                                 WBufferRenderer::RedirectOpenGLContextDefaultFrameBufferObject);
         Q_ASSERT(buffer == helper->bufferRenderer()->currentBuffer());
+        if (!buffer) {
+#ifdef ENABLE_VULKAN_RENDER
+            if (!helper->extraState() && helper->noteVulkanRenderFailed())
+                continue;
+#endif
+        }
+#ifdef ENABLE_VULKAN_RENDER
+        else {
+            helper->noteVulkanRenderSucceeded();
+        }
+#endif
         if (buffer) {
             helper->render(helper->bufferRenderer(), 0, renderMatrix,
                            helper->output()->effectiveSourceRect(),
@@ -1635,8 +1691,13 @@ void WOutputRenderWindowPrivate::doRender(qw_output *needsFrameOutput,
     if (doCommit) {
         committedOutputs.reserve(needsCommit.size());
         for (auto i : std::as_const(needsCommit)) {
+            bool commitAttempted = false;
+            bool commitSucceeded = false;
+            const bool hadCurrentBuffer = i.second->currentBuffer();
             if (Q_UNLIKELY(!i.first->framePending())) {
-                if (Q_LIKELY(i.first->commit(i.second))) {
+                commitAttempted = true;
+                commitSucceeded = i.first->commit(i.second);
+                if (Q_LIKELY(commitSucceeded)) {
                     // Make sure the output is still valid after commit
                     auto output = i.first->output()->output();
                     if (Q_LIKELY(needsFrameOutput)) {
@@ -1649,11 +1710,17 @@ void WOutputRenderWindowPrivate::doRender(qw_output *needsFrameOutput,
                 }
             }
 
-            if (i.second->currentBuffer()) {
+            if (hadCurrentBuffer) {
                 i.second->endRender();
             }
 
             i.first->resetState();
+#ifdef ENABLE_VULKAN_RENDER
+            if (hadCurrentBuffer && commitAttempted && !commitSucceeded
+                && i.first->noteVulkanRenderFailed()) {
+                i.first->update();
+            }
+#endif
         }
     }
 

waylib/src/server/qtquick/wrenderhelper.cpp

@@ -177,6 +177,17 @@ static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC resolveGlEGLImageTargetTexture2DOES()
     return proc;
 }
 
+static void clearGlErrors()
+{
+    while (glGetError() != GL_NO_ERROR) {
+    }
+}
+
+static bool textureUploadSucceeded()
+{
+    return glGetError() == GL_NO_ERROR;
+}
+
 Q_GLOBAL_STATIC(QMutex, s_pendingNativeTextureCleanupMutex)
 Q_GLOBAL_STATIC(QVector<WRenderHelper::NativeTextureCleanup>, s_pendingNativeTextureCleanups)
 
@@ -334,12 +345,25 @@ static bool eglImportDmabufToGLTexture(EGLDisplay display,
     }
 
     GLuint tex = 0;
+    clearGlErrors();
     glGenTextures(1, &tex);
+    if (!tex) {
+        if (auto destroyImage = resolveEglDestroyImageKHR())
+            destroyImage(display, image);
+        return false;
+    }
+
     glBindTexture(GL_TEXTURE_2D, tex);
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
     glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, image);
     glBindTexture(GL_TEXTURE_2D, 0);
+    if (!textureUploadSucceeded()) {
+        glDeleteTextures(1, &tex);
+        if (auto destroyImage = resolveEglDestroyImageKHR())
+            destroyImage(display, image);
+        return false;
+    }
 
     *outImage = image;
     *outTex = tex;
@@ -1433,7 +1457,11 @@ bool WRenderHelper::makeTexture(QRhi *rhi, qw_texture *handle,
             return false;
 
         GLuint glTex = 0;
+        clearGlErrors();
         glGenTextures(1, &glTex);
+        if (!glTex)
+            return false;
+
         glBindTexture(GL_TEXTURE_2D, glTex);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
@@ -1444,6 +1472,10 @@ bool WRenderHelper::makeTexture(QRhi *rhi, qw_texture *handle,
         glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, size.width(), size.height(), 0,
                      GL_BGRA, GL_UNSIGNED_BYTE, pixels.constData());
         glBindTexture(GL_TEXTURE_2D, 0);
+        if (!textureUploadSucceeded()) {
+            glDeleteTextures(1, &glTex);
+            return false;
+        }
 
         texture->setTextureFromNativeTexture(rhi, glTex, 0, 0, size, {},
                                               QQuickWindowPrivate::TextureFromNativeTextureFlags{});

waylib/src/server/qtquick/wsurfaceitem.cpp

@@ -223,6 +223,10 @@ class Q_DECL_HIDDEN WSurfaceItemContentPrivate: public QQuickItemPrivate
             surface = nullptr;
         }
         textureDirty = true;
+#ifdef ENABLE_VULKAN_RENDER
+        textureRetryBackoffFrames = 0;
+        textureRetryDelayFrames = 0;
+#endif
 
         if (frameDoneConnection)
             QObject::disconnect(frameDoneConnection);
@@ -333,6 +337,37 @@ class Q_DECL_HIDDEN WSurfaceItemContentPrivate: public QQuickItemPrivate
         q->setImplicitSize(s.width(), s.height());
     }
 
+#ifdef ENABLE_VULKAN_RENDER
+    inline bool isVulkanRenderer() const {
+        auto rw = qobject_cast<WOutputRenderWindow *>(window);
+        return rw && rw->renderer() && wlr_renderer_is_vk(rw->renderer()->handle());
+    }
+
+    inline bool shouldDeferTextureRetry(bool hasCachedTexture) {
+        if (!hasCachedTexture || !isVulkanRenderer() || textureRetryDelayFrames <= 0)
+            return false;
+
+        --textureRetryDelayFrames;
+        return true;
+    }
+
+    inline void noteTextureUpdateResult(bool ok) {
+        if (!isVulkanRenderer())
+            return;
+
+        if (ok) {
+            textureRetryBackoffFrames = 0;
+            textureRetryDelayFrames = 0;
+            return;
+        }
+
+        textureRetryBackoffFrames = textureRetryBackoffFrames
+            ? qMin(textureRetryBackoffFrames * 2, 4)
+            : 1;
+        textureRetryDelayFrames = textureRetryBackoffFrames;
+    }
+#endif
+
     inline void swapBufferIfNeeded() {
         if (pendingBuffer) {
             buffer = std::move(pendingBuffer);
@@ -377,6 +412,10 @@ class Q_DECL_HIDDEN WSurfaceItemContentPrivate: public QQuickItemPrivate
     bool ignoreBufferOffset = false;
     bool lastRendered = false;
     QAtomicInteger<bool> rendered = false;
+#ifdef ENABLE_VULKAN_RENDER
+    int textureRetryBackoffFrames = 0;
+    int textureRetryDelayFrames = 0;
+#endif
 };
 
 
@@ -516,6 +555,10 @@ void WSurfaceItemContent::setLive(bool live)
     if (live) {
         d->swapBufferIfNeeded();
         d->textureDirty = true;
+#ifdef ENABLE_VULKAN_RENDER
+        d->textureRetryBackoffFrames = 0;
+        d->textureRetryDelayFrames = 0;
+#endif
         update();
     }
     Q_EMIT liveChanged();
@@ -602,16 +645,25 @@ QSGNode *WSurfaceItemContent::updatePaintNode(QSGNode *oldNode, UpdatePaintNodeD
     W_D(WSurfaceItemContent);
 
     auto tp = wTextureProvider();
-    if ((d->live && d->textureDirty) || !tp->texture()) {
+    const bool hasCachedTexture = tp->texture();
+    if (((d->live && d->textureDirty) || !hasCachedTexture)
+#ifdef ENABLE_VULKAN_RENDER
+        && !d->shouldDeferTextureRetry(hasCachedTexture)
+#endif
+    ) {
         auto texture = d->surface ? d->surface->handle()->get_texture() : nullptr;
         bool textureUpdated = false;
         if (texture) {
             textureUpdated = tp->setTexture(qw_texture::from(texture), d->buffer.get());
         } else {
             textureUpdated = tp->setBuffer(d->buffer.get());
         }
-        if (textureUpdated)
+#ifdef ENABLE_VULKAN_RENDER
+        d->noteTextureUpdateResult(textureUpdated);
+#endif
+        if (textureUpdated) {
             d->textureDirty = false;
+        }
     }
 
     if (!tp->texture() || width() <= 0 || height() <= 0) {