Merge pull request 'Improve handling for shader compilation' (#65) from Shader_compilation into master

Reviewed-on: https://git.madvoxel.net/OpenVulkano/OpenVulkano/pulls/65 Reviewed-by: Georg Hagen <georg.hagen@madvoxel.com>
2024-07-15 10:01:45 +02:00
parent 29bad103e6 2a791e5a65
commit 2584d852c8
12 changed files with 145 additions and 1315 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -0,0 +1,3 @@
+# Set line endings to LF, even on Windows. Otherwise, execution within Docker fails.
+# See https://help.github.com/articles/dealing-with-line-endings/
+*.sh text eol=lf
--- a/.gitea/workflows/build_pc.yml
+++ b/.gitea/workflows/build_pc.yml
@@ -43,7 +43,9 @@ jobs:
          cache: true
      - name: Install Dev Packages
        if: matrix.config.os == 'ubuntu-latest'
-        run: sudo apt update && sudo apt install -y extra-cmake-modules libwayland-dev libxkbcommon-dev xorg-dev libarchive-dev libassimp-dev ninja-build
+        run: >
+          sudo apt update && sudo apt install -y extra-cmake-modules libwayland-dev libxkbcommon-dev xorg-dev libarchive-dev libassimp-dev ninja-build glslang-tools glslang-dev unzip
+          && sudo wget https://sourceforge.net/projects/bin2c/files/1.1/bin2c-1.1.zip && sudo unzip bin2c-1.1.zip && cd bin2c && sudo gcc -o bin2c bin2c.c && sudo mv bin2c /usr/bin
      - name: Configure CMake
        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DDEPENDENCY_MIRROR_FILE="${{github.workspace}}/.gitea/workflows/DependencyMirrors.txt"
      - name: Build
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,9 @@
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839

+# Ignore generated shader's output
+openVulkanoCpp/GeneratedShaderData/*
+
 # User-specific stuff
 .idea/**/workspace.xml
 .idea/**/tasks.xml
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -7,6 +7,7 @@ include(cmake/SetCompilerSettings.cmake)
 include(cmake/SetupVulkan.cmake)
 include(cmake/Filter.cmake)
 include(cmake/AppleHelper.cmake)
+include(cmake/SetShaderDependency.cmake)

 set(DEPENDENCY_MIRROR_FILE "DependencyMirrors.txt" CACHE STRING "Dependency mirror")
 VarsFromFile("${DEPENDENCY_MIRROR_FILE}") # Load mirror list (for CICD)
@@ -29,6 +30,15 @@ if(NOT IOS)
 endif()
 SetOptimisationSettings()

+set(BASH_EXECUTABLE "" CACHE FILEPATH "Path to bash executable")
+if ("${BASH_EXECUTABLE}" STREQUAL "")
+    find_program(BASH_PROGRAM bash NO_CACHE REQUIRED)
+    set(BASH_EXECUTABLE ${BASH_PROGRAM} CACHE FILEPATH "Path to bash executable" FORCE)
+endif()
+set(ROOT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR})
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/TryCompileShaders.cmake.in ${CMAKE_BINARY_DIR}/TryCompileShaders.cmake @ONLY)
+execute_process(COMMAND ${BASH_EXECUTABLE} CompileShaders.sh WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/Scripts)
+
 if(IOS)
    set(CMAKE_Swift_LANGUAGE_VERSION 5.0)
    enable_language(Swift)
@@ -79,6 +89,9 @@ if (WIN32)
 endif ()

 SetupVulkan(openVulkanoCpp)
+SetShaderDependency(openVulkanoCpp 
+                    ${CMAKE_CURRENT_SOURCE_DIR}/openVulkanoCpp/Shader
+                    ${CMAKE_CURRENT_SOURCE_DIR}/openVulkanoCpp/GeneratedShaderData)

 if (NOT ANDROID AND NOT IOS)
    target_link_libraries(openVulkanoCpp PRIVATE glfw pugixml)
--- a/README.md
+++ b/README.md
@@ -5,7 +5,11 @@ openVulkano is a simple Vulkan rendering engine which is capable of recording co
 ### Requirements:
 * Git
 * CMake
-* C++ 17 compatible compiler
+* C++ 20 compatible compiler
+* Perl
+* Bash
+* glslang
+* bin2c
 ### Required dependencies on linux
 ```bash
 sudo apt install extra-cmake-modules libarchive-dev libeigen3-dev
--- a/Scripts/CompileShaders.sh
+++ b/Scripts/CompileShaders.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+GENERATED_DATA_PATH="$(dirname "$SCRIPT_DIR")/openVulkanoCpp/GeneratedShaderData"
+SHADER_PATH="$(dirname "$SCRIPT_DIR")/openVulkanoCpp/Shader"
+
+if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
+  SCRIPT_DIR=$(cygpath -w "$SCRIPT_DIR")
+  GENERATED_DATA_PATH=$(cygpath -w "$GENERATED_DATA_PATH")
+  SHADER_PATH=$(cygpath -w "$SHADER_PATH")
+fi
+
+check_return_code() {
+  # return code of last executed command  
+  retVal=$?
+  if [ $retVal -ne 0 ]; then
+    exit 1
+  fi
+}
+
+rm -f *.spv
+rm -f "$GENERATED_DATA_PATH"/*
+mkdir "$GENERATED_DATA_PATH"
+touch "$GENERATED_DATA_PATH"/GeneratedShaderData.h
+touch "$GENERATED_DATA_PATH"/GeneratedShaderData.c
+
+echo "// Auto generated file" >> "$GENERATED_DATA_PATH"/GeneratedShaderData.h
+echo -e "#ifndef GeneratedShaderData_H_INCLUDED\n#define GeneratedShaderData_H_INCLUDED" >> "$GENERATED_DATA_PATH"/GeneratedShaderData.h
+#echo -e "extern \"C\" {" >> "$GENERATED_DATA_PATH"/GeneratedShaderData.h
+echo -e "#include \"GeneratedShaderData.h\"" >> "$GENERATED_DATA_PATH"/GeneratedShaderData.c
+
+# put data from all files with same name into single file
+unique_file_names=($(find $SHADER_PATH/ -type f ! -name '*.hpp' ! -name '*.cpp' | awk -F/ '{print $NF}' | awk -F. '{print $1}' | sort | uniq))
+len=${#unique_file_names[@]}
+i=0
+entriesCountDef="\nconst unsigned int* entriesCount[${len}] = {"
+entriesDef="const void* entries[${len}] = {"
+entriesCountDecl="extern const unsigned int* entriesCount[${len}];"
+entriesDecl="extern const void* entries[${len}];"
+for unique_name in ${unique_file_names[@]} ; do
+    same_files=$(find $SHADER_PATH/ -type f -name "${unique_name}.*")
+    for same_file in ${same_files[@]} ; do
+        filename_with_ext_no_path=$(basename -- "$same_file")
+        glslangValidator -V $same_file -o $filename_with_ext_no_path.spv
+        check_return_code
+    done
+    
+    bin2c -t -d "$GENERATED_DATA_PATH"/$unique_name.h -o "$GENERATED_DATA_PATH"/$unique_name.c *.spv
+    check_return_code
+    rm *.spv
+    
+    echo -e "#include \"${unique_name}.h\"" >> "$GENERATED_DATA_PATH"/GeneratedShaderData.c
+    filename_no_ext_first_upper="${unique_name^}"
+    entriesCountDef+="&fileTable${filename_no_ext_first_upper}Size"
+    entriesDef+="fileTable${filename_no_ext_first_upper}"
+    
+    if [ $i -ne $((len-1)) ]; then
+        entriesCountDef+=",\n\t\t"
+        entriesDef+=",\n\t\t"
+    fi
+    
+    sed -i "3s/.*/#include \"${unique_name}.h\"/" "$GENERATED_DATA_PATH"/$unique_name.c
+    sed -i 's/char \*entryName/const char *entryName/' "$GENERATED_DATA_PATH"/$unique_name.h
+    sed -i 's/const char \*data/const unsigned char *data/' "$GENERATED_DATA_PATH"/$unique_name.h
+    # avoid ODR violation
+    sed -i "s/fileTable/fileTable${filename_no_ext_first_upper}/" "$GENERATED_DATA_PATH"/$unique_name.h $GENERATED_DATA_PATH/$unique_name.c
+    sed -i "s/typedef struct {/#ifndef TFileTableEntry_H_INCLUDED\n#define TFileTableEntry_H_INCLUDED\ntypedef struct {/" "$GENERATED_DATA_PATH"/$unique_name.h
+    sed -i "s/} TFileTableEntry;/} TFileTableEntry;\n#endif \/*TFileTableEntry_H_INCLUDED*\//" "$GENERATED_DATA_PATH"/$unique_name.h
+    ((i++))
+done
+
+entriesCountDef+=" };"
+entriesDef+=" };"
+echo -e "${entriesCountDecl}\n${entriesDecl}\\n#endif /*GeneratedShaderData_H_INCLUDED*/" >> "$GENERATED_DATA_PATH"/GeneratedShaderData.h
+echo -e "\n${entriesCountDef}\n${entriesDef}" >> "$GENERATED_DATA_PATH"/GeneratedShaderData.c
--- a/cmake/SetShaderDependency.cmake
+++ b/cmake/SetShaderDependency.cmake
@@ -0,0 +1,25 @@
+function(SetShaderDependency TARGET SHADER_SRC_FOLDER SHADER_OUTPUT_FOLDER)
+    if (NOT EXISTS ${SHADER_OUTPUT_FOLDER})
+        file(MAKE_DIRECTORY ${SHADER_OUTPUT_FOLDER})
+    endif ()
+
+    file(GLOB SHADER_SRC_FILES ${SHADER_SRC_FOLDER}/*)
+    list(FILTER SHADER_SRC_FILES EXCLUDE REGEX ".*\\.(hpp|cpp)$")
+    foreach(SHADER_FILE ${SHADER_SRC_FILES})
+        get_filename_component(FILENAME ${SHADER_FILE} NAME_WLE)
+        if (NOT ${FILENAME} IN_LIST PROCESSED_FILES)
+            list(APPEND PROCESSED_FILES ${FILENAME})
+            list(APPEND OUTPUT_HEADERS ${SHADER_OUTPUT_FOLDER}/${FILENAME}.h)
+            list(APPEND OUTPUT_SOURCES ${SHADER_OUTPUT_FOLDER}/${FILENAME}.c)
+        endif()
+    endforeach()
+    set(SHADER_EXPECTED_OUTPUT ${OUTPUT_HEADERS} ${OUTPUT_SOURCES})
+
+    add_custom_command(OUTPUT ${SHADER_EXPECTED_OUTPUT}
+                        COMMAND ${CMAKE_COMMAND} -DROOT_FOLDER=${CMAKE_SOURCE_DIR} -P ${CMAKE_BINARY_DIR}/TryCompileShaders.cmake
+                        DEPENDS ${SHADER_SRC_FILES}
+                        COMMENT "Recompiling shaders")
+    add_custom_target(RecompileShaders ALL
+                        DEPENDS ${SHADER_EXPECTED_OUTPUT})
+    add_dependencies(${TARGET} RecompileShaders)
+endfunction(SetShaderDependency)
--- a/cmake/TryCompileShaders.cmake.in
+++ b/cmake/TryCompileShaders.cmake.in
@@ -0,0 +1,4 @@
+execute_process(COMMAND "@BASH_EXECUTABLE@" CompileShaders.sh WORKING_DIRECTORY "@ROOT_FOLDER@/Scripts" RESULT_VARIABLE res_var)
+if(NOT "${res_var}" STREQUAL "0")
+  message(FATAL_ERROR "Failed to recompile shaders")
+endif()
--- a/openVulkanoCpp/Shader/CompileShaders.sh
+++ b/openVulkanoCpp/Shader/CompileShaders.sh
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-rm Shaders.h
-rm Shaders.c
-rm *.spv
-
-for f in ./* ; do
-  if [[ "$f" != *.sh ]] && [[ "$f" != *.hpp ]] && [[ "$f" != *.cpp ]]
-  then
-    glslang -V $f -o $f.spv
-  fi
-done
-
-bin2c -t -d Shaders.h -o Shaders.c *.spv
-
-sed -i 's/char \*entryName/const char *entryName/' Shaders.h
-sed -i 's/const char \*data/const unsigned char *data/' Shaders.h
-
-rm *.spv
--- a/openVulkanoCpp/Shader/ShaderRegistry.cpp
+++ b/openVulkanoCpp/Shader/ShaderRegistry.cpp
@@ -5,8 +5,11 @@
 */

 #include "ShaderRegistry.hpp"
-extern "C" {
-	#include "Shaders.h"
+#include <cassert>
+
+extern "C"
+{
+#include "GeneratedShaderData/GeneratedShaderData.h"
 }

 namespace OpenVulkano
@@ -19,7 +22,12 @@ namespace OpenVulkano

 	ShaderRegistry::ShaderRegistry()
 	{
-		RegisterShaderTable(reinterpret_cast<const ShaderTableEntry*>(fileTable), fileTableSize);
+		assert(std::size(entriesCount) == std::size(entries));
+		const unsigned int sz = std::size(entriesCount);
+		for (unsigned int i = 0; i < sz; i++)
+		{
+			RegisterShaderTable(reinterpret_cast<const ShaderTableEntry*>(entries[i]), *entriesCount[i]);
+		}
 	}

 	void ShaderRegistry::RegisterShaderTable(const ShaderTableEntry* table, uint32_t size)
--- a/openVulkanoCpp/Shader/Shaders.c
+++ b/openVulkanoCpp/Shader/Shaders.c
--- a/openVulkanoCpp/Shader/Shaders.h
+++ b/openVulkanoCpp/Shader/Shaders.h
@@ -1,43 +0,0 @@
-/* Generated by bin2c, do not edit manually */
-#ifndef __Shaders_h_included
-#define __Shaders_h_included
-
-/* Contents of file background.frag.spv */
-extern const long int background_frag_spv_size;
-extern const unsigned char background_frag_spv[1076];
-
-/* Contents of file background.vert.spv */
-extern const long int background_vert_spv_size;
-extern const unsigned char background_vert_spv[4180];
-
-/* Contents of file basic.frag.spv */
-extern const long int basic_frag_spv_size;
-extern const unsigned char basic_frag_spv[376];
-
-/* Contents of file basicTexture.frag.spv */
-extern const long int basicTexture_frag_spv_size;
-extern const unsigned char basicTexture_frag_spv[668];
-
-/* Contents of file basic.vert.spv */
-extern const long int basic_vert_spv_size;
-extern const unsigned char basic_vert_spv[2972];
-
-/* Contents of file grid.frag.spv */
-extern const long int grid_frag_spv_size;
-extern const unsigned char grid_frag_spv[6472];
-
-/* Contents of file grid.vert.spv */
-extern const long int grid_vert_spv_size;
-extern const unsigned char grid_vert_spv[3340];
-
-/* File table comprising original file name, address and size */
-typedef struct {
-	const char *entryName;
-	const unsigned char *data;
-	long int size;
-} TFileTableEntry;
-
-extern const unsigned int fileTableSize;
-extern const TFileTableEntry fileTable[];
-
-#endif    /* __Shaders_h_included */