Add GStreamer support for video textures and update related files

- Introduced GStreamerTexture class for handling video frames.
- Updated texture loading methods to include FromGStreamer.
- Modified effect handling to support GStreamer textures.
- Updated launch configuration and added new sample scripts.

.vscode/launch.json

@@ -7,7 +7,7 @@
             "request": "launch",
             "program": "${workspaceFolder}/build/debug/live-wallpaper",
             "args": [
-                "${workspaceFolder}/samples/dry_rocky_gorge.lua"
+                "${workspaceFolder}/samples/test.lua"
             ],
             "cwd": "${workspaceFolder}",
             "environment": [],

CMakeLists.txt

@@ -64,6 +64,13 @@ add_subdirectory(external/glad)
 find_package(OpenGL REQUIRED)
 find_package(X11 REQUIRED)
 
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(GST REQUIRED IMPORTED_TARGET
+    gstreamer-1.0
+    gstreamer-app-1.0
+    gstreamer-video-1.0
+)
+
 # ── Source Files ─────────────────────────────────────────────────────────────
 file(GLOB_RECURSE SOURCE_FILES
     src/*.cpp
@@ -84,6 +91,7 @@ target_link_libraries(${PROJECT_NAME}
         sol2
         stb
         glad
+        PkgConfig::GST
 )
 
 target_include_directories(${PROJECT_NAME}

rdoc.cap

@@ -0,0 +1,28 @@
+{
+    "rdocCaptureSettings": 1,
+    "settings": {
+        "autoStart": false,
+        "commandLine": "/media/diego/Data/Projects/live-wallpaper/samples/test.lua",
+        "environment": [
+        ],
+        "executable": "/home/diego/Projects/live-wallpaper/build/live-wallpaper",
+        "inject": false,
+        "numQueuedFrames": 1,
+        "options": {
+            "allowFullscreen": true,
+            "allowVSync": true,
+            "apiValidation": false,
+            "captureAllCmdLists": false,
+            "captureCallstacks": false,
+            "captureCallstacksOnlyDraws": false,
+            "debugOutputMute": true,
+            "delayForDebugger": 0,
+            "hookIntoChildren": false,
+            "refAllResources": false,
+            "softMemoryLimit": 0,
+            "verifyBufferAccess": false
+        },
+        "queuedFrameCap": 200,
+        "workingDir": ""
+    }
+}

samples/dry_rocky_gorge.lua

@@ -6,7 +6,7 @@ effect = nil
 rockTexture = nil
 
 function _create()
-    rockTexture = Texture.LoadFromFile("rock_texture.jpg")
+    rockTexture = Texture.FromFile("rock_texture.jpg")
     rockTexture:Bind()
     rockTexture:SetFilter(TextureFilter.LinearMipmapLinear, TextureFilter.Linear)
     rockTexture:SetWrap(TextureWrap.Repeat, TextureWrap.Repeat)

samples/test.lua

@@ -0,0 +1,29 @@
+effect = nil
+video = nil
+
+function _create()
+    video = Texture.FromGStreamer("filesrc location=/media/diego/Data/Projects/live-wallpaper/samples/video.mp4")
+
+    local fxSrc = [[in vec2 vPosition;
+
+        uniform sampler2D uTexture;
+
+        void main() {
+            vec2 uv = vPosition * 0.5 + 0.5;
+            uv.y = 1.0 - uv.y;
+            FragColor = texture(uTexture, uv);
+        }
+    ]]
+    effect = Effect.new(fxSrc)
+end
+
+function _update(dt)
+
+end
+
+function _render()
+    gl.Clear(0, 0, 0, 1.0)
+    effect:Use()
+    effect:SetTexture("uTexture", video, 0)
+    effect:Render()
+end

src/effect.cpp

@@ -9,6 +9,7 @@
 #include "globals.h"
 
 #include "texture.h"
+#include "gstreamer_texture.h"
 
 GLuint Effect::g_dummyVAO = 0;
 
@@ -107,16 +108,18 @@ GLuint Effect::GetUniformLocation(const std::string& name)
     return location;
 }
 
-void Effect::SetSampler(const std::string &name, GLuint textureUnit)
+void Effect::SetTexture(const std::string &name, const Texture& texture, size_t textureUnit)
 {
-    glUniform1i(GetUniformLocation(name), textureUnit);
-}
-
-void Effect::SetTexture(const std::string &name, const Texture &texture, size_t textureUnit)
-{
-    texture.Bind();
     glActiveTexture(GL_TEXTURE0 + static_cast<GLuint>(textureUnit));
-    SetSampler(name, static_cast<GLuint>(textureUnit));
+    texture.Bind();
+    
+    GStreamerTexture* gstTexture = dynamic_cast<GStreamerTexture*>(const_cast<Texture*>(&texture));
+    if (gstTexture) {
+        // For GStreamerTexture, we need to call Update() to upload the latest frame data
+        gstTexture->Update();
+    }
+
+    glUniform1i(GetUniformLocation(name), static_cast<GLint>(textureUnit));
 }
 
 void Effect::SetFloat(const std::string &name, float value)

src/effect.h

@@ -18,7 +18,6 @@ public:
     void Use() const;
     GLuint GetUniformLocation(const std::string& name);
 
-    void SetSampler(const std::string& name, GLuint textureUnit);
     void SetTexture(const std::string& name, const Texture& texture, size_t textureUnit);
     void SetFloat(const std::string& name, float value);
     void SetVector2(const std::string& name, const Vector2& value);

src/gstreamer_texture.cpp

@@ -0,0 +1,170 @@
+#include "gstreamer_texture.h"
+
+#include <stdexcept>
+#include <iostream>
+
+// Callback triggered every time the appsink receives a new frame
+static GstFlowReturn onNewSample(GstAppSink *sink, gpointer user_data) {
+    GstSample *sample = gst_app_sink_pull_sample(sink);
+    if (!sample) return GST_FLOW_ERROR;
+
+    auto *texture = static_cast<GStreamerTexture*>(user_data);
+
+    GstCaps *caps = gst_sample_get_caps(sample);
+    if (caps) {
+        GstStructure *structure = gst_caps_get_structure(caps, 0);
+        int width = 0, height = 0;
+        
+        // Safely extract the integers for width and height
+        if (gst_structure_get_int(structure, "width", &width) &&
+            gst_structure_get_int(structure, "height", &height)) {
+            texture->Resize(static_cast<uint32_t>(width), static_cast<uint32_t>(height));
+        }
+    }
+
+    GstBuffer *buffer = gst_sample_get_buffer(sample);
+    GstMapInfo map;
+
+    // Map the buffer to access the raw pixel memory
+    if (gst_buffer_map(buffer, &map, GST_MAP_READ)) {
+
+        // map.data contains the raw RGBA pixels
+        // map.size contains the total byte size of the frame
+        texture->SendFrame(std::vector<uint8_t>(map.data, map.data + map.size));
+
+        gst_buffer_unmap(buffer, &map);
+    }
+
+    gst_sample_unref(sample);
+    return GST_FLOW_OK;
+}
+
+struct BusCallbackData {
+    GstElement* pipeline;
+    GMainLoop* loop;
+};
+
+// Callback for the GStreamer message bus to handle looping and errors
+static gboolean onBusMessage(GstBus *bus, GstMessage *message, gpointer user_data) {
+    auto *data = static_cast<BusCallbackData*>(user_data);
+
+    switch (GST_MESSAGE_TYPE(message)) {
+        case GST_MESSAGE_EOS:
+            g_print("End of stream reached. Looping back to start...\n");
+            gst_element_seek_simple(
+                data->pipeline,
+                GST_FORMAT_TIME,
+                static_cast<GstSeekFlags>(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT), 0
+            );
+            break;
+        case GST_MESSAGE_ERROR: {
+            GError *err;
+            gchar *debug;
+            gst_message_parse_error(message, &err, &debug);
+            g_printerr("Error: %s\n", err->message);
+            g_error_free(err);
+            g_free(debug);
+            g_main_loop_quit(data->loop);
+            break;
+        }
+        default:
+            break;
+    }
+    return TRUE;
+}
+
+GStreamerTexture::GStreamerTexture(const std::string &pipelineDescription)
+    : Texture2D()
+{
+    if (!gst_is_initialized()) {
+        gst_init(nullptr, nullptr);
+    }
+
+    // append appsink to the pipeline description
+    std::string fullPipelineDescription =
+        pipelineDescription +
+        " ! decodebin ! videoconvert ! video/x-raw,format=RGBA"
+        " ! appsink name=appsink emit-signals=true sync=true";
+
+    GError* error = nullptr;
+    m_pipeline = gst_parse_launch(fullPipelineDescription.c_str(), &error);
+    if (!m_pipeline) {
+        std::string errorMessage = "Failed to create GStreamer pipeline: ";
+        if (error) {
+            errorMessage += error->message;
+            g_clear_error(&error);
+        }
+        throw std::runtime_error(errorMessage);
+    }
+
+    GstAppSink* appSink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(m_pipeline), "appsink"));
+    if (!appSink) {
+        gst_object_unref(m_pipeline);
+        throw std::runtime_error("Failed to get appsink from GStreamer pipeline");
+    }
+
+    g_signal_connect(appSink, "new-sample", G_CALLBACK(onNewSample), this);
+    gst_object_unref(appSink);
+
+    m_loop = g_main_loop_new(nullptr, FALSE);
+
+    auto *busData = new BusCallbackData{m_pipeline, m_loop};
+    GstBus* bus = gst_element_get_bus(m_pipeline);
+    gst_bus_add_signal_watch(bus);
+    g_signal_connect(bus, "message", G_CALLBACK(onBusMessage), busData);
+    gst_object_unref(bus);
+
+    gst_element_set_state(m_pipeline, GST_STATE_PLAYING);
+    m_running = true;
+
+    m_thread = std::thread([this, busData]() {
+        g_main_loop_run(m_loop);
+        m_running = false;
+        delete busData;
+    });
+}
+
+GStreamerTexture::~GStreamerTexture() {
+    if (m_loop) {
+        g_main_loop_quit(m_loop);
+    }
+
+    if (m_thread.joinable()) {
+        m_thread.join();
+    }
+
+    if (m_pipeline) {
+        gst_element_set_state(m_pipeline, GST_STATE_NULL);
+        gst_object_unref(m_pipeline);
+    }
+
+    if (m_loop) {
+        g_main_loop_unref(m_loop);
+    }
+}
+
+void GStreamerTexture::SendFrame(const std::vector<uint8_t>& frameData)
+{
+    std::lock_guard<std::mutex> lock(m_frameMutex);
+    m_frameData = frameData;
+    m_frameAvailable = true;
+}
+
+void GStreamerTexture::Update()
+{
+    std::lock_guard<std::mutex> lock(m_frameMutex);
+    if (m_frameAvailable) {
+        LoadData(TextureFormatType::RGBA8, m_frameData.data());
+        m_frameAvailable = false;
+    }
+}
+
+void GStreamerTexture::Resize(uint32_t width, uint32_t height)
+{
+    if (m_width == width && m_height == height) {
+        return; // No need to resize if dimensions are the same
+    }
+
+    m_width = width;
+    m_height = height;
+}

src/gstreamer_texture.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "texture.h"
+
+extern "C" {
+#include <gst/gst.h>
+#include <gst/app/gstappsink.h>
+}
+
+#include <thread>
+#include <atomic>
+#include <mutex>
+#include <vector>
+#include <string>
+
+class GStreamerTexture : public Texture2D {
+public:
+    GStreamerTexture(const std::string& pipelineDescription);
+    ~GStreamerTexture() override;
+
+    void SendFrame(const std::vector<uint8_t>& frameData);
+    void Update();
+
+    void Resize(uint32_t width, uint32_t height);
+
+private:
+    GstElement* m_pipeline = nullptr;
+    GMainLoop* m_loop = nullptr;
+    std::thread m_thread;
+    std::atomic<bool> m_running{false};
+    mutable std::mutex m_frameMutex;
+
+protected:
+    std::vector<uint8_t> m_frameData;
+    bool m_frameAvailable = false;
+};

src/lua.cpp

@@ -4,6 +4,7 @@
 
 #include "tmath.hpp"
 #include "texture.h"
+#include "gstreamer_texture.h"
 #include "framebuffer.h"
 #include "effect.h"
 #include "opengl.h"
@@ -303,18 +304,36 @@ static void RegisterTexture(sol::state& lua) {
         "GetID", &TextureCubeMap::GetID
     );
 
+    // ── GStreamerTexture ─────────────────────────────────────────────────
+    lua.new_usertype<GStreamerTexture>("GStreamerTexture",
+        sol::no_constructor,
+        sol::base_classes, sol::bases<Texture>(),
+        "Bind", &GStreamerTexture::Bind,
+        "Unbind", &GStreamerTexture::Unbind,
+        "GenerateMipmaps", &GStreamerTexture::GenerateMipmaps,
+        "SetFilter", &GStreamerTexture::SetFilter,
+        "SetWrap", sol::overload(
+            sol::resolve<void(TextureWrap, TextureWrap) const>(&GStreamerTexture::SetWrap),
+            sol::resolve<void(TextureWrap, TextureWrap, TextureWrap) const>(&GStreamerTexture::SetWrap)
+        ),
+        "GetWidth", &GStreamerTexture::GetWidth,
+        "GetHeight", &GStreamerTexture::GetHeight,
+        "GetID", &GStreamerTexture::GetID
+    );
+
     // ── Texture static factories ────────────────────────────────────────
     auto tex = lua.create_named_table("Texture");
     tex["CreateTexture1D"] = static_cast<std::unique_ptr<Texture1D>(*)(uint32_t, TextureFormatType)>(&Texture::CreateTexture);
     tex["CreateTexture2D"] = static_cast<std::unique_ptr<Texture2D>(*)(uint32_t, uint32_t, TextureFormatType)>(&Texture::CreateTexture);
     tex["CreateTexture3D"] = static_cast<std::unique_ptr<Texture3D>(*)(uint32_t, uint32_t, uint32_t, TextureFormatType)>(&Texture::CreateTexture);
     tex["CreateCubeMap"] = &Texture::CreateCubeMap;
-    tex["LoadFromFile"] = [](const std::string& filepath) -> std::unique_ptr<Texture2D> {
+    tex["FromGStreamer"] = &Texture::FromGStreamer;
+    tex["FromFile"] = [](const std::string& filepath) -> std::unique_ptr<Texture2D> {
         std::filesystem::path p(filepath);
         if (p.is_relative() && !g_ScriptDir.empty()) {
             p = std::filesystem::path(g_ScriptDir) / p;
         }
-        return Texture::LoadFromFile(p.string());
+        return Texture::FromFile(p.string());
     };
 }
 
@@ -347,7 +366,6 @@ static void RegisterEffect(sol::state& lua) {
     auto fx = lua.new_usertype<Effect>("Effect",
         sol::constructors<Effect(), Effect(const std::string&)>(),
         "Use", &Effect::Use,
-        "SetSampler", &Effect::SetSampler,
         "SetTexture", &Effect::SetTexture,
         "SetFloat", &Effect::SetFloat,
         "SetVector2", &Effect::SetVector2,

src/texture.cpp

@@ -1,4 +1,5 @@
 #include "texture.h"
+#include "gstreamer_texture.h"
 
 #include "stb_image.h"
 #include <stdexcept>
@@ -128,7 +129,12 @@ std::unique_ptr<TextureCubeMap> Texture::CreateCubeMap(uint32_t size, TextureFor
     return tex;
 }
 
-std::unique_ptr<Texture2D> Texture::LoadFromFile(const std::string &filepath)
+std::unique_ptr<GStreamerTexture> Texture::FromGStreamer(const std::string &pipeline)
+{
+    return std::make_unique<GStreamerTexture>(pipeline);
+}
+
+std::unique_ptr<Texture2D> Texture::FromFile(const std::string &filepath)
 {
     int width, height, channels;
     stbi_set_flip_vertically_on_load(true);
@@ -161,6 +167,10 @@ Texture::Texture(GLenum target)
     : m_target(target)
 {
     glGenTextures(1, &m_id);
+    glBindTexture(m_target, m_id);
+    glTexParameteri(m_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(m_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glBindTexture(m_target, 0);
 }
 
 Texture::~Texture()
@@ -213,41 +223,80 @@ void TextureCubeMap::LoadFaceDataFromFile(Face face, const std::string &filepath
     stbi_image_free(data);
 }
 
-void Texture1D::LoadData(TextureFormatType format, const void *data) const
+void Texture1D::LoadData(TextureFormatType format, const void *data)
 {
     const TextureFormat& fmt = textureFormatMap[static_cast<int>(format)];
-    glTexImage1D(
-        m_target,
-        0,
-        fmt.internalFormat,
-        m_width, 0,
-        fmt.format, fmt.type,
-        data
-    );
+    if (!m_hasInitialData) {
+        // Initialize texture storage with null data to allocate GPU memory
+        glTexImage1D(
+            m_target,
+            0,
+            fmt.internalFormat,
+            m_width, 0,
+            fmt.format, fmt.type,
+            nullptr
+        );
+        m_hasInitialData = true;
+    } else {
+        // Update existing texture data
+        glTexSubImage1D(
+            m_target,
+            0,
+            0, m_width,
+            fmt.format, fmt.type,
+            data
+        );
+    }
 }
 
-void Texture2D::LoadData(TextureFormatType format, const void *data) const
+void Texture2D::LoadData(TextureFormatType format, const void *data)
 {
     const TextureFormat& fmt = textureFormatMap[static_cast<int>(format)];
-    glTexImage2D(
-        m_target,
-        0,
-        fmt.internalFormat,
-        m_width, m_height, 0,
-        fmt.format, fmt.type,
-        data
-    );
+    if (!m_hasInitialData) {
+        // Initialize texture storage and optionally upload data
+        glTexImage2D(
+            m_target,
+            0,
+            fmt.internalFormat,
+            m_width, m_height, 0,
+            fmt.format, fmt.type,
+            data
+        );
+        m_hasInitialData = true;
+    } else {
+        // Update existing texture data
+        glTexSubImage2D(
+            m_target,
+            0,
+            0, 0, m_width, m_height,
+            fmt.format, fmt.type,
+            data
+        );
+    }
 }
 
-void Texture3D::LoadData(TextureFormatType format, const void *data) const
+void Texture3D::LoadData(TextureFormatType format, const void *data)
 {
     const TextureFormat& fmt = textureFormatMap[static_cast<int>(format)];
-    glTexImage3D(
-        m_target,
-        0,
-        fmt.internalFormat,
-        m_width, m_height, m_depth, 0,
-        fmt.format, fmt.type,
-        data
-    );
+    if (!m_hasInitialData) {
+        // Initialize texture storage with null data to allocate GPU memory
+        glTexImage3D(
+            m_target,
+            0,
+            fmt.internalFormat,
+            m_width, m_height, m_depth, 0,
+            fmt.format, fmt.type,
+            nullptr
+        );
+        m_hasInitialData = true;
+    } else {
+        // Update existing texture data
+        glTexSubImage3D(
+            m_target,
+            0,
+            0, 0, 0, m_width, m_height, m_depth,
+            fmt.format, fmt.type,
+            data
+        );
+    }
 }

src/texture.h

@@ -43,6 +43,7 @@ class Texture1D;
 class Texture2D;
 class Texture3D;
 class TextureCubeMap;
+class GStreamerTexture;
 
 // RAII wrapper base class for OpenGL textures
 class Texture : public IGPUObject<GLuint> {
@@ -62,15 +63,17 @@ public:
     static std::unique_ptr<Texture2D> CreateTexture(uint32_t width, uint32_t height, TextureFormatType format);
     static std::unique_ptr<Texture3D> CreateTexture(uint32_t width, uint32_t height, uint32_t depth, TextureFormatType format);
     static std::unique_ptr<TextureCubeMap> CreateCubeMap(uint32_t size, TextureFormatType format);
-    static std::unique_ptr<Texture2D> LoadFromFile(const std::string& filepath);
+    static std::unique_ptr<GStreamerTexture> FromGStreamer(const std::string& pipeline);
+    static std::unique_ptr<Texture2D> FromFile(const std::string& filepath);
 
 protected:
     GLenum m_target;
+    bool m_hasInitialData = false;
 
     virtual void LoadData(
         TextureFormatType format,
         const void* data
-    ) const = 0;
+    ) = 0;
 };
 
 class Texture1D : public Texture {
@@ -86,7 +89,7 @@ protected:
     void LoadData(
         TextureFormatType format,
         const void* data
-    ) const override;
+    ) override;
 };
 
 class Texture2D : public Texture {
@@ -105,7 +108,7 @@ protected:
     void LoadData(
         TextureFormatType format,
         const void* data
-    ) const override;
+    ) override;
 };
 
 class Texture3D : public Texture {
@@ -125,7 +128,7 @@ protected:
     void LoadData(
         TextureFormatType format,
         const void* data
-    ) const override;
+    ) override;
 };
 
 class TextureCubeMap : public Texture {
@@ -161,5 +164,5 @@ protected:
     void LoadData(
         TextureFormatType format,
         const void* data
-    ) const override {}
+    ) override {}
 };