diff --git a/.vscode/launch.json b/.vscode/launch.json index 7378dc0..5cca8aa 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,7 @@ "request": "launch", "program": "${workspaceFolder}/build/debug/live-wallpaper", "args": [ - "${workspaceFolder}/samples/dry_rocky_gorge.lua" + "${workspaceFolder}/samples/test.lua" ], "cwd": "${workspaceFolder}", "environment": [], diff --git a/CMakeLists.txt b/CMakeLists.txt index ab16afe..b586e5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,13 @@ add_subdirectory(external/glad) find_package(OpenGL REQUIRED) find_package(X11 REQUIRED) +find_package(PkgConfig REQUIRED) +pkg_check_modules(GST REQUIRED IMPORTED_TARGET + gstreamer-1.0 + gstreamer-app-1.0 + gstreamer-video-1.0 +) + # ── Source Files ───────────────────────────────────────────────────────────── file(GLOB_RECURSE SOURCE_FILES src/*.cpp @@ -84,6 +91,7 @@ target_link_libraries(${PROJECT_NAME} sol2 stb glad + PkgConfig::GST ) target_include_directories(${PROJECT_NAME} diff --git a/rdoc.cap b/rdoc.cap new file mode 100644 index 0000000..af5b316 --- /dev/null +++ b/rdoc.cap @@ -0,0 +1,28 @@ +{ + "rdocCaptureSettings": 1, + "settings": { + "autoStart": false, + "commandLine": "/media/diego/Data/Projects/live-wallpaper/samples/test.lua", + "environment": [ + ], + "executable": "/home/diego/Projects/live-wallpaper/build/live-wallpaper", + "inject": false, + "numQueuedFrames": 1, + "options": { + "allowFullscreen": true, + "allowVSync": true, + "apiValidation": false, + "captureAllCmdLists": false, + "captureCallstacks": false, + "captureCallstacksOnlyDraws": false, + "debugOutputMute": true, + "delayForDebugger": 0, + "hookIntoChildren": false, + "refAllResources": false, + "softMemoryLimit": 0, + "verifyBufferAccess": false + }, + "queuedFrameCap": 200, + "workingDir": "" + } +} diff --git a/samples/dry_rocky_gorge.lua b/samples/dry_rocky_gorge.lua index aeb4a90..9a07670 100644 --- a/samples/dry_rocky_gorge.lua +++ b/samples/dry_rocky_gorge.lua @@ -6,7 +6,7 @@ effect = nil rockTexture = nil function _create() - rockTexture = Texture.LoadFromFile("rock_texture.jpg") + rockTexture = Texture.FromFile("rock_texture.jpg") rockTexture:Bind() rockTexture:SetFilter(TextureFilter.LinearMipmapLinear, TextureFilter.Linear) rockTexture:SetWrap(TextureWrap.Repeat, TextureWrap.Repeat) diff --git a/samples/test.lua b/samples/test.lua new file mode 100644 index 0000000..d154004 --- /dev/null +++ b/samples/test.lua @@ -0,0 +1,29 @@ +effect = nil +video = nil + +function _create() + video = Texture.FromGStreamer("filesrc location=/media/diego/Data/Projects/live-wallpaper/samples/video.mp4") + + local fxSrc = [[in vec2 vPosition; + + uniform sampler2D uTexture; + + void main() { + vec2 uv = vPosition * 0.5 + 0.5; + uv.y = 1.0 - uv.y; + FragColor = texture(uTexture, uv); + } + ]] + effect = Effect.new(fxSrc) +end + +function _update(dt) + +end + +function _render() + gl.Clear(0, 0, 0, 1.0) + effect:Use() + effect:SetTexture("uTexture", video, 0) + effect:Render() +end \ No newline at end of file diff --git a/samples/video.mp4 b/samples/video.mp4 new file mode 100644 index 0000000..9ac25fb Binary files /dev/null and b/samples/video.mp4 differ diff --git a/src/effect.cpp b/src/effect.cpp index d08d97b..4b40f81 100644 --- a/src/effect.cpp +++ b/src/effect.cpp @@ -9,6 +9,7 @@ #include "globals.h" #include "texture.h" +#include "gstreamer_texture.h" GLuint Effect::g_dummyVAO = 0; @@ -107,16 +108,18 @@ GLuint Effect::GetUniformLocation(const std::string& name) return location; } -void Effect::SetSampler(const std::string &name, GLuint textureUnit) +void Effect::SetTexture(const std::string &name, const Texture& texture, size_t textureUnit) { - glUniform1i(GetUniformLocation(name), textureUnit); -} - -void Effect::SetTexture(const std::string &name, const Texture &texture, size_t textureUnit) -{ - texture.Bind(); glActiveTexture(GL_TEXTURE0 + static_cast(textureUnit)); - SetSampler(name, static_cast(textureUnit)); + texture.Bind(); + + GStreamerTexture* gstTexture = dynamic_cast(const_cast(&texture)); + if (gstTexture) { + // For GStreamerTexture, we need to call Update() to upload the latest frame data + gstTexture->Update(); + } + + glUniform1i(GetUniformLocation(name), static_cast(textureUnit)); } void Effect::SetFloat(const std::string &name, float value) diff --git a/src/effect.h b/src/effect.h index f0f9e83..b6c25d7 100644 --- a/src/effect.h +++ b/src/effect.h @@ -18,7 +18,6 @@ public: void Use() const; GLuint GetUniformLocation(const std::string& name); - void SetSampler(const std::string& name, GLuint textureUnit); void SetTexture(const std::string& name, const Texture& texture, size_t textureUnit); void SetFloat(const std::string& name, float value); void SetVector2(const std::string& name, const Vector2& value); diff --git a/src/gstreamer_texture.cpp b/src/gstreamer_texture.cpp new file mode 100644 index 0000000..697d1df --- /dev/null +++ b/src/gstreamer_texture.cpp @@ -0,0 +1,170 @@ +#include "gstreamer_texture.h" + +#include +#include + +// Callback triggered every time the appsink receives a new frame +static GstFlowReturn onNewSample(GstAppSink *sink, gpointer user_data) { + GstSample *sample = gst_app_sink_pull_sample(sink); + if (!sample) return GST_FLOW_ERROR; + + auto *texture = static_cast(user_data); + + GstCaps *caps = gst_sample_get_caps(sample); + if (caps) { + GstStructure *structure = gst_caps_get_structure(caps, 0); + int width = 0, height = 0; + + // Safely extract the integers for width and height + if (gst_structure_get_int(structure, "width", &width) && + gst_structure_get_int(structure, "height", &height)) { + texture->Resize(static_cast(width), static_cast(height)); + } + } + + GstBuffer *buffer = gst_sample_get_buffer(sample); + GstMapInfo map; + + // Map the buffer to access the raw pixel memory + if (gst_buffer_map(buffer, &map, GST_MAP_READ)) { + + // map.data contains the raw RGBA pixels + // map.size contains the total byte size of the frame + texture->SendFrame(std::vector(map.data, map.data + map.size)); + + gst_buffer_unmap(buffer, &map); + } + + gst_sample_unref(sample); + return GST_FLOW_OK; +} + +struct BusCallbackData { + GstElement* pipeline; + GMainLoop* loop; +}; + +// Callback for the GStreamer message bus to handle looping and errors +static gboolean onBusMessage(GstBus *bus, GstMessage *message, gpointer user_data) { + auto *data = static_cast(user_data); + + switch (GST_MESSAGE_TYPE(message)) { + case GST_MESSAGE_EOS: + g_print("End of stream reached. Looping back to start...\n"); + gst_element_seek_simple( + data->pipeline, + GST_FORMAT_TIME, + static_cast(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT), 0 + ); + break; + case GST_MESSAGE_ERROR: { + GError *err; + gchar *debug; + gst_message_parse_error(message, &err, &debug); + g_printerr("Error: %s\n", err->message); + g_error_free(err); + g_free(debug); + g_main_loop_quit(data->loop); + break; + } + default: + break; + } + return TRUE; +} + +GStreamerTexture::GStreamerTexture(const std::string &pipelineDescription) + : Texture2D() +{ + if (!gst_is_initialized()) { + gst_init(nullptr, nullptr); + } + + // append appsink to the pipeline description + std::string fullPipelineDescription = + pipelineDescription + + " ! decodebin ! videoconvert ! video/x-raw,format=RGBA" + " ! appsink name=appsink emit-signals=true sync=true"; + + GError* error = nullptr; + m_pipeline = gst_parse_launch(fullPipelineDescription.c_str(), &error); + if (!m_pipeline) { + std::string errorMessage = "Failed to create GStreamer pipeline: "; + if (error) { + errorMessage += error->message; + g_clear_error(&error); + } + throw std::runtime_error(errorMessage); + } + + GstAppSink* appSink = GST_APP_SINK(gst_bin_get_by_name(GST_BIN(m_pipeline), "appsink")); + if (!appSink) { + gst_object_unref(m_pipeline); + throw std::runtime_error("Failed to get appsink from GStreamer pipeline"); + } + + g_signal_connect(appSink, "new-sample", G_CALLBACK(onNewSample), this); + gst_object_unref(appSink); + + m_loop = g_main_loop_new(nullptr, FALSE); + + auto *busData = new BusCallbackData{m_pipeline, m_loop}; + GstBus* bus = gst_element_get_bus(m_pipeline); + gst_bus_add_signal_watch(bus); + g_signal_connect(bus, "message", G_CALLBACK(onBusMessage), busData); + gst_object_unref(bus); + + gst_element_set_state(m_pipeline, GST_STATE_PLAYING); + m_running = true; + + m_thread = std::thread([this, busData]() { + g_main_loop_run(m_loop); + m_running = false; + delete busData; + }); +} + +GStreamerTexture::~GStreamerTexture() { + if (m_loop) { + g_main_loop_quit(m_loop); + } + + if (m_thread.joinable()) { + m_thread.join(); + } + + if (m_pipeline) { + gst_element_set_state(m_pipeline, GST_STATE_NULL); + gst_object_unref(m_pipeline); + } + + if (m_loop) { + g_main_loop_unref(m_loop); + } +} + +void GStreamerTexture::SendFrame(const std::vector& frameData) +{ + std::lock_guard lock(m_frameMutex); + m_frameData = frameData; + m_frameAvailable = true; +} + +void GStreamerTexture::Update() +{ + std::lock_guard lock(m_frameMutex); + if (m_frameAvailable) { + LoadData(TextureFormatType::RGBA8, m_frameData.data()); + m_frameAvailable = false; + } +} + +void GStreamerTexture::Resize(uint32_t width, uint32_t height) +{ + if (m_width == width && m_height == height) { + return; // No need to resize if dimensions are the same + } + + m_width = width; + m_height = height; +} diff --git a/src/gstreamer_texture.h b/src/gstreamer_texture.h new file mode 100644 index 0000000..83e7326 --- /dev/null +++ b/src/gstreamer_texture.h @@ -0,0 +1,36 @@ +#pragma once + +#include "texture.h" + +extern "C" { +#include +#include +} + +#include +#include +#include +#include +#include + +class GStreamerTexture : public Texture2D { +public: + GStreamerTexture(const std::string& pipelineDescription); + ~GStreamerTexture() override; + + void SendFrame(const std::vector& frameData); + void Update(); + + void Resize(uint32_t width, uint32_t height); + +private: + GstElement* m_pipeline = nullptr; + GMainLoop* m_loop = nullptr; + std::thread m_thread; + std::atomic m_running{false}; + mutable std::mutex m_frameMutex; + +protected: + std::vector m_frameData; + bool m_frameAvailable = false; +}; \ No newline at end of file diff --git a/src/lua.cpp b/src/lua.cpp index 6c3d631..7a7b60e 100644 --- a/src/lua.cpp +++ b/src/lua.cpp @@ -4,6 +4,7 @@ #include "tmath.hpp" #include "texture.h" +#include "gstreamer_texture.h" #include "framebuffer.h" #include "effect.h" #include "opengl.h" @@ -303,18 +304,36 @@ static void RegisterTexture(sol::state& lua) { "GetID", &TextureCubeMap::GetID ); + // ── GStreamerTexture ───────────────────────────────────────────────── + lua.new_usertype("GStreamerTexture", + sol::no_constructor, + sol::base_classes, sol::bases(), + "Bind", &GStreamerTexture::Bind, + "Unbind", &GStreamerTexture::Unbind, + "GenerateMipmaps", &GStreamerTexture::GenerateMipmaps, + "SetFilter", &GStreamerTexture::SetFilter, + "SetWrap", sol::overload( + sol::resolve(&GStreamerTexture::SetWrap), + sol::resolve(&GStreamerTexture::SetWrap) + ), + "GetWidth", &GStreamerTexture::GetWidth, + "GetHeight", &GStreamerTexture::GetHeight, + "GetID", &GStreamerTexture::GetID + ); + // ── Texture static factories ──────────────────────────────────────── auto tex = lua.create_named_table("Texture"); tex["CreateTexture1D"] = static_cast(*)(uint32_t, TextureFormatType)>(&Texture::CreateTexture); tex["CreateTexture2D"] = static_cast(*)(uint32_t, uint32_t, TextureFormatType)>(&Texture::CreateTexture); tex["CreateTexture3D"] = static_cast(*)(uint32_t, uint32_t, uint32_t, TextureFormatType)>(&Texture::CreateTexture); tex["CreateCubeMap"] = &Texture::CreateCubeMap; - tex["LoadFromFile"] = [](const std::string& filepath) -> std::unique_ptr { + tex["FromGStreamer"] = &Texture::FromGStreamer; + tex["FromFile"] = [](const std::string& filepath) -> std::unique_ptr { std::filesystem::path p(filepath); if (p.is_relative() && !g_ScriptDir.empty()) { p = std::filesystem::path(g_ScriptDir) / p; } - return Texture::LoadFromFile(p.string()); + return Texture::FromFile(p.string()); }; } @@ -347,7 +366,6 @@ static void RegisterEffect(sol::state& lua) { auto fx = lua.new_usertype("Effect", sol::constructors(), "Use", &Effect::Use, - "SetSampler", &Effect::SetSampler, "SetTexture", &Effect::SetTexture, "SetFloat", &Effect::SetFloat, "SetVector2", &Effect::SetVector2, diff --git a/src/texture.cpp b/src/texture.cpp index 33b84d4..d88d291 100644 --- a/src/texture.cpp +++ b/src/texture.cpp @@ -1,4 +1,5 @@ #include "texture.h" +#include "gstreamer_texture.h" #include "stb_image.h" #include @@ -128,7 +129,12 @@ std::unique_ptr Texture::CreateCubeMap(uint32_t size, TextureFor return tex; } -std::unique_ptr Texture::LoadFromFile(const std::string &filepath) +std::unique_ptr Texture::FromGStreamer(const std::string &pipeline) +{ + return std::make_unique(pipeline); +} + +std::unique_ptr Texture::FromFile(const std::string &filepath) { int width, height, channels; stbi_set_flip_vertically_on_load(true); @@ -161,6 +167,10 @@ Texture::Texture(GLenum target) : m_target(target) { glGenTextures(1, &m_id); + glBindTexture(m_target, m_id); + glTexParameteri(m_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(m_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glBindTexture(m_target, 0); } Texture::~Texture() @@ -213,41 +223,80 @@ void TextureCubeMap::LoadFaceDataFromFile(Face face, const std::string &filepath stbi_image_free(data); } -void Texture1D::LoadData(TextureFormatType format, const void *data) const +void Texture1D::LoadData(TextureFormatType format, const void *data) { const TextureFormat& fmt = textureFormatMap[static_cast(format)]; - glTexImage1D( - m_target, - 0, - fmt.internalFormat, - m_width, 0, - fmt.format, fmt.type, - data - ); + if (!m_hasInitialData) { + // Initialize texture storage with null data to allocate GPU memory + glTexImage1D( + m_target, + 0, + fmt.internalFormat, + m_width, 0, + fmt.format, fmt.type, + nullptr + ); + m_hasInitialData = true; + } else { + // Update existing texture data + glTexSubImage1D( + m_target, + 0, + 0, m_width, + fmt.format, fmt.type, + data + ); + } } -void Texture2D::LoadData(TextureFormatType format, const void *data) const +void Texture2D::LoadData(TextureFormatType format, const void *data) { const TextureFormat& fmt = textureFormatMap[static_cast(format)]; - glTexImage2D( - m_target, - 0, - fmt.internalFormat, - m_width, m_height, 0, - fmt.format, fmt.type, - data - ); + if (!m_hasInitialData) { + // Initialize texture storage and optionally upload data + glTexImage2D( + m_target, + 0, + fmt.internalFormat, + m_width, m_height, 0, + fmt.format, fmt.type, + data + ); + m_hasInitialData = true; + } else { + // Update existing texture data + glTexSubImage2D( + m_target, + 0, + 0, 0, m_width, m_height, + fmt.format, fmt.type, + data + ); + } } -void Texture3D::LoadData(TextureFormatType format, const void *data) const +void Texture3D::LoadData(TextureFormatType format, const void *data) { const TextureFormat& fmt = textureFormatMap[static_cast(format)]; - glTexImage3D( - m_target, - 0, - fmt.internalFormat, - m_width, m_height, m_depth, 0, - fmt.format, fmt.type, - data - ); + if (!m_hasInitialData) { + // Initialize texture storage with null data to allocate GPU memory + glTexImage3D( + m_target, + 0, + fmt.internalFormat, + m_width, m_height, m_depth, 0, + fmt.format, fmt.type, + nullptr + ); + m_hasInitialData = true; + } else { + // Update existing texture data + glTexSubImage3D( + m_target, + 0, + 0, 0, 0, m_width, m_height, m_depth, + fmt.format, fmt.type, + data + ); + } } diff --git a/src/texture.h b/src/texture.h index d3962a8..0b51c1c 100644 --- a/src/texture.h +++ b/src/texture.h @@ -43,6 +43,7 @@ class Texture1D; class Texture2D; class Texture3D; class TextureCubeMap; +class GStreamerTexture; // RAII wrapper base class for OpenGL textures class Texture : public IGPUObject { @@ -62,15 +63,17 @@ public: static std::unique_ptr CreateTexture(uint32_t width, uint32_t height, TextureFormatType format); static std::unique_ptr CreateTexture(uint32_t width, uint32_t height, uint32_t depth, TextureFormatType format); static std::unique_ptr CreateCubeMap(uint32_t size, TextureFormatType format); - static std::unique_ptr LoadFromFile(const std::string& filepath); + static std::unique_ptr FromGStreamer(const std::string& pipeline); + static std::unique_ptr FromFile(const std::string& filepath); protected: GLenum m_target; + bool m_hasInitialData = false; virtual void LoadData( TextureFormatType format, const void* data - ) const = 0; + ) = 0; }; class Texture1D : public Texture { @@ -86,7 +89,7 @@ protected: void LoadData( TextureFormatType format, const void* data - ) const override; + ) override; }; class Texture2D : public Texture { @@ -105,7 +108,7 @@ protected: void LoadData( TextureFormatType format, const void* data - ) const override; + ) override; }; class Texture3D : public Texture { @@ -125,7 +128,7 @@ protected: void LoadData( TextureFormatType format, const void* data - ) const override; + ) override; }; class TextureCubeMap : public Texture { @@ -161,5 +164,5 @@ protected: void LoadData( TextureFormatType format, const void* data - ) const override {} + ) override {} }; \ No newline at end of file