Merge pull request 'Building CURL on windows properly & tests for WebResourceLoader' (#150) from curl_webresourceloader into master

Reviewed-on: https://git.madvoxel.net/OpenVulkano/OpenVulkano/pulls/150
Reviewed-by: Georg Hagen <georg.hagen@madvoxel.com>

3rdParty/curl/CMakeLists.txt

@@ -1,27 +1,28 @@
 include(Utils)
+include(FetchContent)
+
+set(CURL_REPO https://github.com/curl/curl.git)
+set(CURL_GIT_TAG curl-8_8_0)
 
 set(CURL_DEPS_INSTALL ${CMAKE_BINARY_DIR}/deps_curl)
 find_package(CURL QUIET)
 if (NOT ${CURL_FOUND})
-    file(MAKE_DIRECTORY ${CURL_DEPS_INSTALL})
+    if (NOT WIN32)
-    if (WIN32)
+        file(MAKE_DIRECTORY ${CURL_DEPS_INSTALL})
-        set(EXT_DIR ext_windows)
+        execute_process(
-    else ()
+                COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} -DTOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/ext -DCURL_REPO=${CURL_REPO} -DCURL_GIT_TAG=${CURL_GIT_TAG} -DOPENSSL_REPO=${OPENSSL_REPO} -DPLATFORM=${PLATFORM}
-        set(EXT_DIR ext)
+                WORKING_DIRECTORY ${CURL_DEPS_INSTALL}
-    endif ()
+        )
-    execute_process(
+        execute_process(
-            COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} -DTOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/${EXT_DIR} -DCURL_REPO=${CURL_REPO} -DOPENSSL_REPO=${OPENSSL_REPO} -DPLATFORM=${PLATFORM}
+                COMMAND ${CMAKE_COMMAND} --build ${CURL_DEPS_INSTALL} --config Release
-            WORKING_DIRECTORY ${CURL_DEPS_INSTALL}
+                RESULT_VARIABLE build_result
-    )
+        )
-    execute_process(
+        if (NOT ${build_result} EQUAL "0")
-            COMMAND ${CMAKE_COMMAND} --build ${CURL_DEPS_INSTALL} --config Release
+            message(FATAL_ERROR "Failed to build curl!")
-            RESULT_VARIABLE build_result
+        endif()
-    )
-    if (NOT ${build_result} EQUAL "0")
-        message(FATAL_ERROR "Failed to build curl!")
-    endif()
 
-    list(APPEND CMAKE_PREFIX_PATH ${CURL_DEPS_INSTALL}/INSTALL)
+        list(APPEND CMAKE_PREFIX_PATH ${CURL_DEPS_INSTALL}/INSTALL)
+    endif ()
 else ()
     message("Using system curl")
     set(USING_SYSTEM_CURL ON PARENT_SCOPE)
@@ -38,12 +39,31 @@ function(LinkCurl TARGET)
         if (APPLE)
             target_link_libraries(${TARGET} PUBLIC curl)
         elseif (WIN32)
-            target_link_libraries(${TARGET} PUBLIC ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
+            set(CURL_USE_SCHANNEL ON CACHE BOOL "USE SCHANNEL")
+            FetchContent_Declare(
+                curl
+                # In function you cannot access the variables defined outside, so hardcoding for now.
+                GIT_REPOSITORY https://github.com/curl/curl.git
+                GIT_TAG curl-8_8_0
+                GIT_SHALLOW TRUE
+                CMAKE_ARGS
+                    -DCMAKE_BUILD_TYPE=Release
+                    -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}/INSTALL
+                    -DUSE_NGHTTP2=OFF
+                    -DBUILD_STATIC_LIBS=ON
+                    -DBUILD_SHARED_LIBS=OFF
+                OVERRIDE_FIND_PACKAGE
+            )
+            FetchContent_MakeAvailable(curl)
+            find_package(curl REQUIRED)
+            target_link_libraries(${TARGET} PUBLIC CURL::libcurl)
         else ()
             target_link_libraries(${TARGET} PUBLIC crypto ssl curl)
         endif ()
     else ()
-        target_include_directories(${TARGET} PUBLIC ${CURL_INCLUDE_DIR})
+        if (CURL_INCLUDE_DIR)
+            target_include_directories(${TARGET} PUBLIC ${CURL_INCLUDE_DIR})
+        endif ()
         target_link_libraries(${TARGET} PUBLIC CURL::libcurl)
     endif ()
     target_compile_definitions(${TARGET} PRIVATE HAS_CURL)

3rdParty/curl/ext/CMakeLists.txt

@@ -6,6 +6,9 @@ include(FetchContent)
 if (NOT DEFINED CURL_REPO OR CURL_REPO STREQUAL "")
     set(CURL_REPO https://github.com/curl/curl.git)
 endif ()
+if (NOT DEFINED CURL_GIT_TAG OR CURL_GIT_TAG STREQUAL "")
+    set(CURL_GIT_TAG curl-8_8_0)
+endif ()
 if (NOT DEFINED OPENSSL_REPO OR OPENSSL_REPO STREQUAL "")
     set(OPENSSL_REPO https://github.com/openssl/openssl.git)
 endif ()
@@ -29,8 +32,6 @@ else ()
     set(OPENSSL_MAKE_COMMAND make)
 endif ()
 
-set(CURL_GIT_TAG curl-8_8_0)
-
 function(InstallOpenSSL)
     ExternalProject_Add(
             OpenSSL

3rdParty/curl/ext_windows/CMakeLists.txt

@@ -1,45 +0,0 @@
-cmake_minimum_required(VERSION 3.16)
-project(build_curl)
-include(FetchContent)
-
-if (NOT DEFINED CURL_MIRROR_LINK)
-    set(CURL_MIRROR_LINK https://curl.se/download/curl-8.8.0.tar.gz)
-endif ()
-if (NOT DEFINED LIBRESSL_MIRROR_LINK)
-    set(LIBRESSL_MIRROR_LINK https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-3.9.2.tar.gz)
-endif ()
-
-set(LIBRESSL_TESTS OFF CACHE BOOL "" FORCE)
-FetchContent_Declare(
-    libressl
-    URL ${LIBRESSL_MIRROR_LINK}
-    CMAKE_ARGS
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}/INSTALL
-)
-FetchContent_MakeAvailable(libressl)
-
-set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries" FORCE)
-set(BUILD_STATIC_LIBS ON CACHE BOOL "Build static libraries" FORCE)
-set(BUILD_STATIC_CURL ON CACHE BOOL "Build curl executable with static libcurl" FORCE)
-set(CURL_DISABLE_TESTS ON CACHE BOOL "Do no build tests" FORCE)
-
-set(CURL_USE_LIBRESSL ON CACHE BOOL "Use LibreSSL instead of OpenSSL" FORCE)
-set(OPENSSL_ROOT_DIR ${libressl_SOURCE_DIR} CACHE PATH "Path to LibreSSL root directory" FORCE)
-if (WIN32)
-    set(OPENSSL_CRYPTO_LIBRARY ${libressl_BINARY_DIR}/crypto/release/crypto.lib CACHE FILEPATH "Path to LibreSSL crypto library" FORCE)
-    set(OPENSSL_SSL_LIBRARY ${libressl_BINARY_DIR}/ssl/release/ssl.lib CACHE FILEPATH "Path to LibreSSL SSL library" FORCE)
-else ()
-    set(OPENSSL_CRYPTO_LIBRARY ${libressl_BINARY_DIR}/crypto/libcrypto.a CACHE FILEPATH "Path to LibreSSL crypto library" FORCE)
-    set(OPENSSL_SSL_LIBRARY ${libressl_BINARY_DIR}/ssl/libssl.a CACHE FILEPATH "Path to LibreSSL SSL library" FORCE)
-endif ()
-set(OPENSSL_INCLUDE_DIR ${libressl_SOURCE_DIR}/include CACHE PATH "Path to LibreSSL include directory" FORCE)
-
-FetchContent_Declare(
-    curl
-    URL ${CURL_MIRROR_LINK}
-    CMAKE_ARGS
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}/INSTALL
-)
-FetchContent_MakeAvailable(curl)

openVulkanoCpp/Host/WebResourceLoader.cpp

@@ -81,8 +81,9 @@ namespace OpenVulkano
 			CURLcode result = curl_easy_perform(curl);
 			if (result != CURLE_OK)
 			{
+				std::string error = curl_easy_strerror(result);
+				Logger::APP->error("Failed to download resource: '" + url + "' - " + error);
 				curl_easy_cleanup(curl);
-				Logger::APP->error("Failed to download resource: '" + url + "' - " + curl_easy_strerror(result));
 				return Array<char>();
 			}
 			curl_easy_cleanup(curl);
@@ -92,6 +93,7 @@ namespace OpenVulkano
 		std::ofstream file(cacheFilePath, std::ios::binary);
 		file.write(buffer.data(), buffer.size());
 #endif
+
 		return Array<char>(buffer);
 	}
 

openVulkanoCpp/Host/WebResourceLoader.hpp

@@ -14,6 +14,7 @@ namespace OpenVulkano
 {
 	class WebResourceLoader : public ResourceLoader
 	{
+	protected:
 		std::filesystem::path m_cacheDirectory;
 
 		std::filesystem::path GetCacheFilePath(const std::string& url);

tests/Host/WebResourceLoader.cpp

@@ -0,0 +1,137 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+#include <catch2/catch_all.hpp>
+
+#include "Host/WebResourceLoader.hpp"
+
+#include <filesystem>
+#include <fstream>
+#include <cstring>
+
+#include "curl/curl.h"
+
+using namespace OpenVulkano;
+
+class TestWebResourceLoader : public WebResourceLoader
+{
+public:
+	std::filesystem::path GetCacheFilePath(const std::string& url)
+	{
+		return WebResourceLoader::GetCacheFilePath(url);
+	}
+
+	Array<char> DownloadResource(const std::string& url)
+	{
+		return WebResourceLoader::DownloadResource(url);
+	}
+
+	std::filesystem::path GetCacheDir()
+	{
+		return m_cacheDirectory;
+	}
+};
+
+TEST_CASE("CURL SSL support", "[WebResourceLoader]")
+{
+	curl_version_info_data* vinfo = curl_version_info(CURLVERSION_NOW);
+	REQUIRE(vinfo->features & CURL_VERSION_SSL);
+}
+
+TEST_CASE("Constructor/Destructor", "[WebResourceLoader]")
+{
+	TestWebResourceLoader loader;
+
+	REQUIRE(std::filesystem::exists(loader.GetCacheDir()));
+}
+
+TEST_CASE("IsUrl", "[WebResourceLoader]")
+{
+	REQUIRE(WebResourceLoader::IsUrl("http://example.com"));
+	REQUIRE(WebResourceLoader::IsUrl("https://example.com"));
+	REQUIRE(WebResourceLoader::IsUrl("ftp://example.com"));
+	REQUIRE_FALSE(WebResourceLoader::IsUrl("file://example.com"));
+	REQUIRE_FALSE(WebResourceLoader::IsUrl("example.com"));
+}
+
+TEST_CASE("GetCacheFilePath", "[WebResourceLoader]")
+{
+	TestWebResourceLoader loader;
+
+	std::string url = "http://example.com/resource";
+	std::filesystem::path cachePath = loader.GetCacheFilePath(url);
+
+	size_t expectedHash = std::hash<std::string> {}(url);
+	std::string expectedHashStr = std::to_string(expectedHash);
+
+	REQUIRE(cachePath.filename().string() == expectedHashStr);
+	REQUIRE(cachePath.parent_path().filename().string() == "resources");
+}
+
+TEST_CASE("DownloadResource from non-ssl uri", "[WebResourceLoader]")
+{
+	TestWebResourceLoader loader;
+	std::string url = "http://neverssl.com";
+	Array<char> resourceData = loader.DownloadResource(url);
+	REQUIRE(!resourceData.Empty());
+	std::filesystem::path cachePath = loader.GetCacheFilePath(url);
+	REQUIRE(std::filesystem::exists(cachePath));
+	std::filesystem::remove(cachePath);
+}
+
+TEST_CASE("DownloadResource with curl", "[WebResourceLoader]")
+{
+	TestWebResourceLoader loader;
+	std::string url = "https://example.com/resource";
+	Array<char> resourceData = loader.DownloadResource(url);
+	REQUIRE(!resourceData.Empty());
+	std::filesystem::path cachePath = loader.GetCacheFilePath(url);
+	REQUIRE(std::filesystem::exists(cachePath));
+	std::filesystem::remove(cachePath);
+}
+
+TEST_CASE("DownloadResource without curl", "[WebResourceLoader]")
+{
+	TestWebResourceLoader loader;
+	std::string url = "https://example.com/resource";
+	Array<char> resourceData = loader.DownloadResource(url);
+
+	REQUIRE(!resourceData.Empty());
+}
+
+TEST_CASE("GetResource", "[WebResourceLoader]")
+{
+	TestWebResourceLoader loader;
+
+	std::string url = "https://example.com/resource";
+
+	{
+		std::filesystem::path cachePath = loader.GetCacheFilePath(url);
+		std::ofstream file(cachePath, std::ios::binary);
+		std::string mockContent = "Mock cached content";
+		file.write(mockContent.c_str(), mockContent.size());
+		file.close();
+
+		Array<char> resource = loader.GetResource(url);
+
+		REQUIRE(!resource.Empty());
+		REQUIRE(std::memcmp(resource.Data(), mockContent.c_str(), mockContent.size()) == 0);
+
+		std::filesystem::remove(cachePath);
+	}
+
+	{
+		std::filesystem::path cachePath = loader.GetCacheFilePath(url);
+		std::filesystem::remove(cachePath);
+
+		Array<char> resource = loader.GetResource(url);
+
+		REQUIRE(!resource.Empty());
+		REQUIRE(std::filesystem::exists(cachePath));
+
+		std::filesystem::remove(cachePath);
+	}
+}