From 985ced1a7a9b6ae35eae6a4b6f15809ea866bca6 Mon Sep 17 00:00:00 2001 From: Seth Berrier Date: Wed, 16 Mar 2022 19:28:03 -0500 Subject: [PATCH] All basic code for video sprite (untested) - Have shaders and a video sprite in place - AVDecoder class in place and compiling - Everything compiles, links, and runs - Not using the video sprite in any way yet --- Sonataria/Sonataria.vcxproj | 20 +++++--- Sonataria/Sonataria.vcxproj.filters | 6 +++ Sonataria/VideoShader.cpp | 27 ++++++++++ Sonataria/VideoShader.h | 15 ++++++ Sonataria/VideoSprite.cpp | 79 +++++++++++++++++++++++++++++ Sonataria/VideoSprite.h | 19 ++++++- Sonataria/videoFragment.shader | 18 +++---- Sonataria/videoVertex.shader | 25 ++++++--- 8 files changed, 187 insertions(+), 22 deletions(-) create mode 100644 Sonataria/VideoShader.cpp create mode 100644 Sonataria/VideoShader.h diff --git a/Sonataria/Sonataria.vcxproj b/Sonataria/Sonataria.vcxproj index 0ffacc5..cb0d52a 100644 --- a/Sonataria/Sonataria.vcxproj +++ b/Sonataria/Sonataria.vcxproj @@ -83,8 +83,8 @@ Console - $(SolutionDir)dependencies\FreeImage\x32\;$(SolutionDir)dependencies\SFML\SFML-2.5.1\lib;$(SolutionDir)dependencies\PacDrive\lib;$(SolutionDir)dependencies\glew-2.1.0\lib\Release\Win32;$(SolutionDir)dependencies\FreeType\lib;$(SolutionDir)dependencies\uFCoder\lib\x86;%(AdditionalLibraryDirectories) - FreeImage.lib;opengl32.lib;glu32.lib;glew32.lib;freetype.lib;legacy_stdio_definitions.lib;uFCoder-x86.lib;sfml-system-d.lib;sfml-graphics-d.lib;sfml-audio-d.lib;sfml-network-d.lib;sfml-window-d.lib;PacDrive32.lib;%(AdditionalDependencies);iphlpapi.lib + $(SolutionDir)dependencies\ffmpeg-4.5\lib;$(SolutionDir)dependencies\FreeImage\x32\;$(SolutionDir)dependencies\SFML\SFML-2.5.1\lib;$(SolutionDir)dependencies\PacDrive\lib;$(SolutionDir)dependencies\glew-2.1.0\lib\Release\Win32;$(SolutionDir)dependencies\FreeType\lib;$(SolutionDir)dependencies\uFCoder\lib\x86;%(AdditionalLibraryDirectories) + FreeImage.lib;opengl32.lib;glu32.lib;glew32.lib;freetype.lib;legacy_stdio_definitions.lib;uFCoder-x86.lib;avcodec.lib;avformat.lib;avutil.lib;sfml-system-d.lib;sfml-graphics-d.lib;sfml-audio-d.lib;sfml-network-d.lib;sfml-window-d.lib;PacDrive32.lib;%(AdditionalDependencies);iphlpapi.lib copy $(SolutionDir)dependencies\FreeImage\x32\FreeImage.dll $(OutputPath) @@ -98,7 +98,10 @@ copy $(SolutionDir)dependencies\SFML\SFML-2.5.1\bin\sfml-network-d-2.dll $(Outp copy $(SolutionDir)dependencies\SFML\SFML-2.5.1\bin\sfml-system-d-2.dll $(OutputPath) copy $(SolutionDir)dependencies\SFML\SFML-2.5.1\bin\sfml-window-d-2.dll $(OutputPath) copy $(SolutionDir)dependencies\uFCoder\bin\uFCoder-x86.dll $(OutputPath) - +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\avcodec-59.dll $(OutputPath) +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\avformat-59.dll $(OutputPath) +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\avutil-57.dll $(OutputPath) +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\swresample-4.dll $(OutputPath) Copy Dependent DLLs to output directory @@ -137,8 +140,8 @@ copy $(SolutionDir)dependencies\uFCoder\bin\uFCoder-x86.dll $(OutputPath) Console true true - $(SolutionDir)dependencies\FreeImage\x32\;$(SolutionDir)dependencies\SFML\SFML-2.5.1\lib;$(SolutionDir)dependencies\PacDrive\lib;$(SolutionDir)dependencies\glew-2.1.0\lib\Release\Win32;$(SolutionDir)dependencies\FreeType\lib;$(SolutionDir)dependencies\uFCoder\lib\x86;%(AdditionalLibraryDirectories) - FreeImage.lib;opengl32.lib;glu32.lib;glew32.lib;freetype.lib;legacy_stdio_definitions.lib;uFCoder-x86.lib;sfml-system.lib;sfml-graphics.lib;sfml-audio.lib;sfml-network.lib;sfml-window.lib;%(AdditionalDependencies);PacDrive32.lib;iphlpapi.lib + $(SolutionDir)dependencies\ffmpeg-4.5\lib;$(SolutionDir)dependencies\FreeImage\x32\;$(SolutionDir)dependencies\SFML\SFML-2.5.1\lib;$(SolutionDir)dependencies\PacDrive\lib;$(SolutionDir)dependencies\glew-2.1.0\lib\Release\Win32;$(SolutionDir)dependencies\FreeType\lib;$(SolutionDir)dependencies\uFCoder\lib\x86;%(AdditionalLibraryDirectories) + FreeImage.lib;opengl32.lib;glu32.lib;glew32.lib;freetype.lib;legacy_stdio_definitions.lib;uFCoder-x86.lib;avcodec.lib;avformat.lib;avutil.lib;sfml-system.lib;sfml-graphics.lib;sfml-audio.lib;sfml-network.lib;sfml-window.lib;PacDrive32.lib;iphlpapi.lib;%(AdditionalDependencies) copy $(SolutionDir)dependencies\FreeImage\x32\FreeImage.dll $(OutputPath) @@ -152,7 +155,10 @@ copy $(SolutionDir)dependencies\SFML\SFML-2.5.1\bin\sfml-network-d-2.dll $(Outp copy $(SolutionDir)dependencies\SFML\SFML-2.5.1\bin\sfml-system-d-2.dll $(OutputPath) copy $(SolutionDir)dependencies\SFML\SFML-2.5.1\bin\sfml-window-d-2.dll $(OutputPath) copy $(SolutionDir)dependencies\uFCoder\bin\uFCoder-x86.dll $(OutputPath) - +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\avcodec-59.dll $(OutputPath) +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\avformat-59.dll $(OutputPath) +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\avutil-57.dll $(OutputPath) +copy $(SolutionDir)dependencies\ffmpeg-4.5\bin\swresample-4.dll $(OutputPath) Copy Dependent DLLs to output directory @@ -211,6 +217,7 @@ copy $(SolutionDir)dependencies\uFCoder\bin\uFCoder-x86.dll $(OutputPath) + @@ -249,6 +256,7 @@ copy $(SolutionDir)dependencies\uFCoder\bin\uFCoder-x86.dll $(OutputPath) + diff --git a/Sonataria/Sonataria.vcxproj.filters b/Sonataria/Sonataria.vcxproj.filters index ba25ccc..ed8e4e0 100644 --- a/Sonataria/Sonataria.vcxproj.filters +++ b/Sonataria/Sonataria.vcxproj.filters @@ -120,6 +120,9 @@ Source Files + + Source Files + @@ -230,6 +233,9 @@ Header Files + + Header Files + diff --git a/Sonataria/VideoShader.cpp b/Sonataria/VideoShader.cpp new file mode 100644 index 0000000..1e2a8f7 --- /dev/null +++ b/Sonataria/VideoShader.cpp @@ -0,0 +1,27 @@ +#include "VideoShader.h" + +#include "resource.h" +#include "OpenGLSprite.h" + +VideoShader::VideoShader() : OpenGLShader(IDR_VIDEO_VERTEX_SHADER, IDR_VIDEO_FRAGMENT_SHADER) { + textureLoc[0] = 0; + textureLoc[1] = 0; + textureLoc[2] = 0; +} + +VideoShader::~VideoShader() {} + +void VideoShader::initAttributes() { + glBindAttribLocation(this->shaderProgram, ATTRIB_POSITION_INDEX, "v_pos"); + glBindAttribLocation(this->shaderProgram, ATTRIB_COLOR_INDEX, "v_col"); + glBindAttribLocation(this->shaderProgram, ATTRIB_TEX_UV_INDEX, "v_uv"); +} + +void VideoShader::initUniforms() { + textureLoc[0] = glGetUniformLocation(this->shaderProgram, "f_yTexture"); + textureLoc[1] = glGetUniformLocation(this->shaderProgram, "f_uTexture"); + textureLoc[2] = glGetUniformLocation(this->shaderProgram, "f_vTexture"); + glUniform1i(textureLoc[0], 0); + glUniform1i(textureLoc[1], 1); + glUniform1i(textureLoc[2], 2); +} diff --git a/Sonataria/VideoShader.h b/Sonataria/VideoShader.h new file mode 100644 index 0000000..188e37a --- /dev/null +++ b/Sonataria/VideoShader.h @@ -0,0 +1,15 @@ +#pragma once +#include "OpenGLShader.h" + +class VideoShader : public OpenGLShader +{ +public: + VideoShader(); + virtual ~VideoShader(); + +protected: + GLint textureLoc[3]; + + virtual void initAttributes() override; + virtual void initUniforms() override; +}; diff --git a/Sonataria/VideoSprite.cpp b/Sonataria/VideoSprite.cpp index eaf5168..9565d7e 100644 --- a/Sonataria/VideoSprite.cpp +++ b/Sonataria/VideoSprite.cpp @@ -1 +1,80 @@ #include "VideoSprite.h" + +#include "AVDecode.h" + +VideoSprite::VideoSprite(const std::wstring& newName) : QuadSprite(newName) +{ + myDecoder = new AVDecode(); + videoDone = true; // Until the video is loaded, say it is done +} + +VideoSprite::~VideoSprite() +{ + delete myDecoder; +} + +bool VideoSprite::loadVideo(const char* videoFilename) +{ + // Attempt to initialize the video decoder + if (!myDecoder->prepareToDecode(videoFilename)) + { + return false; + } + videoDone = false; + + // Make the textures that will hold the video frame color planes + const int width = myDecoder->getWidth(); + const int height = myDecoder->getHeight(); + + // Generate the textures in OpenGL state only + glGenTextures(1, &this->textures[0]); + glBindTexture(GL_TEXTURE_2D, this->textures[0]); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, NULL); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + glGenTextures(1, &this->textures[1]); + glBindTexture(GL_TEXTURE_2D, this->textures[1]); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width / 2, height / 2, 0, GL_RED, GL_UNSIGNED_BYTE, NULL); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + glGenTextures(1, &this->textures[2]); + glBindTexture(GL_TEXTURE_2D, this->textures[2]); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width / 2, height / 2, 0, GL_RED, GL_UNSIGNED_BYTE, NULL); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + // Update the decoder textures + myDecoder->setTextureIDs(this->textures[0], this->textures[1], this->textures[2]); + + // Return success + return true; +} + +void VideoSprite::update(int elapsedTimeMillis) +{ + // Attempt to update the video frame + if (!videoDone) { + if (!myDecoder->decodeFrame()) { + myDecoder->closeDecoder(); + videoDone = true; + } + } +} + +void VideoSprite::render(PROJECTION projType) const +{ + // Bind the three texture planes to the first three texture targets + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, this->textures[0]); + + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, this->textures[1]); + + glActiveTexture(GL_TEXTURE2); + glBindTexture(GL_TEXTURE_2D, this->textures[2]); + + // Render the underlying quad + QuadSprite::render(projType); +} diff --git a/Sonataria/VideoSprite.h b/Sonataria/VideoSprite.h index 3f94ad0..8632375 100644 --- a/Sonataria/VideoSprite.h +++ b/Sonataria/VideoSprite.h @@ -1,8 +1,25 @@ #pragma once #include "QuadSprite.h" - +class AVDecode; class VideoSprite : public QuadSprite { +public: + VideoSprite(const std::wstring& newName); + virtual ~VideoSprite(); + + bool loadVideo(const char* videoFilename); + void update(int elapsedTimeMillis); + virtual void render(PROJECTION projType) const; + +protected: + // The video decoder object + AVDecode* myDecoder; + + // Playback state + bool videoDone; + + // Internal texture IDs + GLuint textures[3]; }; diff --git a/Sonataria/videoFragment.shader b/Sonataria/videoFragment.shader index 216f550..f960458 100644 --- a/Sonataria/videoFragment.shader +++ b/Sonataria/videoFragment.shader @@ -1,9 +1,9 @@ #version 400 // The three separate Yuv color planes -uniform sampler2D yTexture; -uniform sampler2D uTexture; -uniform sampler2D vTexture; +uniform sampler2D f_yTexture; +uniform sampler2D f_uTexture; +uniform sampler2D f_vTexture; // Chromaticity values for conversion to RGB const vec3 R_cf = vec3(1.164383, 0.000000, 1.596027); @@ -12,19 +12,19 @@ const vec3 B_cf = vec3(1.164383, 2.017232, 0.000000); const vec3 offset = vec3(-0.0625, -0.5, -0.5); // Main input and output -in vec2 texcoord; -out vec4 fragcolor; +in vec2 f_uv; +out vec4 frag_color; void main() { // Read the individual y, u, and v values - float y = texture(yTexture, texcoord).r; - float u = texture(uTexture, texcoord).r; - float v = texture(vTexture, texcoord).r; + float y = texture(f_yTexture, f_uv).r; + float u = texture(f_uTexture, f_uv).r; + float v = texture(f_vTexture, f_uv).r; // Construct the vector and offset the values vec3 yuv = vec3(y, u, v); yuv += offset; // Multiply by chromaticity constants to generate RGB value - fragcolor = vec4(dot(yuv, R_cf), dot(yuv, G_cf), dot(yuv, B_cf), 1.0); + frag_color = vec4(dot(yuv, R_cf), dot(yuv, G_cf), dot(yuv, B_cf), 1.0); } diff --git a/Sonataria/videoVertex.shader b/Sonataria/videoVertex.shader index a326e96..1d1b96e 100644 --- a/Sonataria/videoVertex.shader +++ b/Sonataria/videoVertex.shader @@ -1,10 +1,23 @@ #version 400 -in vec2 position; -out vec2 texcoord; +// Input variables that can change per-vertex (attribes from openGL) +in vec3 v_pos; // - position of the vertex +in vec3 v_col; // - Color of the vertex +in vec2 v_uv; // - uv coordinates for the vertex -void main() -{ - gl_Position = vec4(position, 0.0, 1.0); - texcoord = position * vec2(0.5) + vec2(0.5); +// Uniform variables (can only change per primitive) +uniform mat4 model; // Model matrix +uniform mat4 projection; // Projection matrix + +// Output to the fragment shader +out vec4 f_col; +out vec4 f_uv; + +void main() { + // Copy in the color and texture attributes + f_uv = vec4(v_uv, 0.0, 1.0); + f_col = vec4(v_col, 1.0); + + // Apply transformation to position and output to fragment + gl_Position = projection * model * vec4(v_pos, 1.0); }