diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 8a00e6a2..629ddc06 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -4,6 +4,11 @@ on: pull_request: branches: [ main ] +env: + # Path to the CMake build directory. + build: '${{ github.workspace }}/build' + config: 'Debug' + jobs: cancel-old-build: @@ -14,6 +19,90 @@ jobs: with: access_token: ${{ github.token }} + msvc-analyze: + name: MSVC Analyze + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Configure CMake + run: cmake -B ${{ env.build }} -DCMAKE_BUILD_TYPE=${{ env.config }} + + # Build is not required unless generated source files are used + # - name: Build CMake + # run: cmake --build ${{ env.build }} --config ${{ env.config }} + + - name: Run MSVC Code Analysis + uses: microsoft/msvc-code-analysis-action@v0.1.1 + # Provide a unique ID to access the sarif output path + id: run-analysis + with: + cmakeBuildDirectory: ${{ env.build }} + buildConfiguration: ${{ env.config }} + # Ruleset file that will determine what checks will be run + ruleset: NativeRecommendedRules.ruleset + # Paths to ignore analysis of CMake targets and includes + # ignoredPaths: ${{ github.workspace }}/dependencies;${{ github.workspace }}/test + + # Upload SARIF file to GitHub Code Scanning Alerts + - name: Upload SARIF to GitHub + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: ${{ steps.run-analysis.outputs.sarif }} + + # Upload SARIF file as an Artifact to download and view + - name: Upload SARIF as an Artifact + uses: actions/upload-artifact@v2 + with: + name: sarif-file + path: ${{ steps.run-analysis.outputs.sarif }} + + codeql-analyze: + name: CodeQL Analyze + runs-on: 'ubuntu-latest' + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: cpp + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # Command-line programs to run using the OS shell. + # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:cpp" + build-linux: runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..aa2fef06 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,35 @@ +# CMakeList.txt : Top-level CMake project file, do global configuration +# and include sub-projects here. +# +cmake_minimum_required (VERSION 3.8) + +# Enable Hot Reload for MSVC compilers if supported. +if (POLICY CMP0141) + cmake_policy(SET CMP0141 NEW) + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "$,$>,$<$:EditAndContinue>,$<$:ProgramDatabase>>") +endif() + +project ("longtail") + +enable_language(C) +enable_language(ASM) +enable_language(CXX) + +#add_compile_definitions(ZSTD_DISABLE_ASM) + +# Include sub-projects. +add_subdirectory("lib") + +add_subdirectory("src") + +add_subdirectory("static_lib") + +#add_subdirectory("shared_lib") + +add_subdirectory ("cmd") + +add_subdirectory("test") + +enable_testing() + +add_test(NAME test COMMAND test_base WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000..351f244e --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,125 @@ +{ + "version": 3, + "configurePresets": [ + { + "name": "windows-gcc-base", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_C_COMPILER": "gcc.exe", + "CMAKE_CXX_COMPILER": "g++.exe" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "windows-base", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_C_COMPILER": "cl.exe", + "CMAKE_CXX_COMPILER": "cl.exe" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "x64-debug", + "displayName": "x64 Debug", + "inherits": "windows-base", + "architecture": { + "value": "x64", + "strategy": "external" + }, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "x64-release", + "displayName": "x64 Release", + "inherits": "x64-debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "x64-gcc-debug", + "displayName": "x64 GCC Debug", + "inherits": "windows-gcc-base", + "architecture": { + "value": "x64", + "strategy": "external" + }, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "x64-gcc-release", + "displayName": "x64 GCC Release", + "inherits": "x64-gcc-debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "linux-debug", + "displayName": "Linux Debug", + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Linux" + }, + "vendor": { + "microsoft.com/VisualStudioRemoteSettings/CMake/1.0": { + "sourceDir": "$env{HOME}/.vs/$ms{projectDirName}" + } + } + }, + { + "name": "macos-debug", + "displayName": "macOS Debug", + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + }, + "vendor": { + "microsoft.com/VisualStudioRemoteSettings/CMake/1.0": { + "sourceDir": "$env{HOME}/.vs/$ms{projectDirName}" + } + } + } + ], + "testPresets": [ + { + "name": "testPresets", + "displayName": "Custom test preset", + "description": "Custom test preset description", + "configurePreset": "x64-debug" + } + ] +} \ No newline at end of file diff --git a/cmd/CMakeLists.txt b/cmd/CMakeLists.txt new file mode 100644 index 00000000..03bd1d37 --- /dev/null +++ b/cmd/CMakeLists.txt @@ -0,0 +1,14 @@ +# CMakeList.txt : CMake project for longtail, include source and define +# project specific logic here. +# + +# Add source to this project's executable. +add_executable (longtail "main.c" "ext/kgflags.h") + +if (CMAKE_VERSION VERSION_GREATER 3.12) + set_property(TARGET longtail PROPERTY CXX_STANDARD 20) +endif() + +# TODO: Add tests and install targets if needed. +target_link_libraries(longtail PRIVATE static_lib) + diff --git a/cmd/main.c b/cmd/main.c index 5df10da9..2f705d48 100644 --- a/cmd/main.c +++ b/cmd/main.c @@ -21,7 +21,7 @@ #include "../lib/meowhash/longtail_meowhash.h" #include "../lib/ratelimitedprogress/longtail_ratelimitedprogress.h" #include "../lib/shareblockstore/longtail_shareblockstore.h" -#include "../lib/brotli/longtail_brotli.h" +//#include "../lib/brotli/longtail_brotli.h" #include "../lib/lz4/longtail_lz4.h" #include "../lib/zstd/longtail_zstd.h" #include "../lib/longtail_platform.h" @@ -172,30 +172,30 @@ uint32_t ParseCompressionType(const char* compression_algorithm) { { return 0; } - if (strcmp("brotli", compression_algorithm) == 0) - { - return Longtail_GetBrotliGenericDefaultQuality(); - } - if (strcmp("brotli_min", compression_algorithm) == 0) - { - return Longtail_GetBrotliGenericMinQuality(); - } - if (strcmp("brotli_max", compression_algorithm) == 0) - { - return Longtail_GetBrotliGenericMaxQuality(); - } - if (strcmp("brotli_text", compression_algorithm) == 0) - { - return Longtail_GetBrotliTextDefaultQuality(); - } - if (strcmp("brotli_text_min", compression_algorithm) == 0) - { - return Longtail_GetBrotliTextMinQuality(); - } - if (strcmp("brotli_text_max", compression_algorithm) == 0) - { - return Longtail_GetBrotliTextMaxQuality(); - } +// if (strcmp("brotli", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliGenericDefaultQuality(); +// } +// if (strcmp("brotli_min", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliGenericMinQuality(); +// } +// if (strcmp("brotli_max", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliGenericMaxQuality(); +// } +// if (strcmp("brotli_text", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliTextDefaultQuality(); +// } +// if (strcmp("brotli_text_min", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliTextMinQuality(); +// } +// if (strcmp("brotli_text_max", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliTextMaxQuality(); +// } if (strcmp("lz4", compression_algorithm) == 0) { return Longtail_GetLZ4DefaultQuality(); diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 00000000..617d2c3c --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,24 @@ +add_library(platform STATIC "longtail_platform.c" "longtail_platform.h") + +add_subdirectory ("archiveblockstore") +add_subdirectory ("atomiccancel") +add_subdirectory ("bikeshed") +add_subdirectory ("blake2") +add_subdirectory ("blake3") +add_subdirectory ("blockstorestorage") +add_subdirectory ("brotli") +add_subdirectory ("cacheblockstore") +add_subdirectory ("compressblockstore") +add_subdirectory ("compressionregistry") +add_subdirectory ("filestorage") +add_subdirectory ("fsblockstore") +add_subdirectory ("hashregistry") +add_subdirectory ("hpcdcchunker") +add_subdirectory ("lrublockstore") +add_subdirectory ("lz4") +add_subdirectory ("memstorage") +add_subdirectory ("memtracer") +add_subdirectory ("meowhash") +add_subdirectory ("ratelimitedprogress") +add_subdirectory ("shareblockstore") +add_subdirectory ("zstd") diff --git a/lib/archiveblockstore/CMakeLists.txt b/lib/archiveblockstore/CMakeLists.txt new file mode 100644 index 00000000..7c987bc0 --- /dev/null +++ b/lib/archiveblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(archiveblockstore OBJECT "longtail_archiveblockstore.c" "longtail_archiveblockstore.h") diff --git a/lib/atomiccancel/CMakeLists.txt b/lib/atomiccancel/CMakeLists.txt new file mode 100644 index 00000000..e81a4d49 --- /dev/null +++ b/lib/atomiccancel/CMakeLists.txt @@ -0,0 +1 @@ +add_library(atomiccancel OBJECT "longtail_atomiccancel.c" "longtail_atomiccancel.h") diff --git a/lib/bikeshed/CMakeLists.txt b/lib/bikeshed/CMakeLists.txt new file mode 100644 index 00000000..2272a116 --- /dev/null +++ b/lib/bikeshed/CMakeLists.txt @@ -0,0 +1 @@ +add_library(bikeshed OBJECT "longtail_bikeshed.c" "longtail_bikeshed.h" "ext/bikeshed.h") diff --git a/lib/blake2/CMakeLists.txt b/lib/blake2/CMakeLists.txt new file mode 100644 index 00000000..dc0cf0fe --- /dev/null +++ b/lib/blake2/CMakeLists.txt @@ -0,0 +1,19 @@ +add_library(blake2 OBJECT + "longtail_blake2.c" + "longtail_blake2.h" + "ext/blake2-config.h" + "ext/blake2-impl.h" + "ext/blake2.h" + "ext/blake2s-load-sse2.h" + "ext/blake2s-load-sse41.h" + "ext/blake2s-round.h" + "ext/blake2s.c" +) + +if(MSVC) + target_compile_options(blake2 INTERFACE /arch:SSE2) + target_compile_options(blake2 PUBLIC -DHAVE_SSE2) +else() + target_compile_options(blake2 PUBLIC -msse2) +endif(MSVC) + \ No newline at end of file diff --git a/lib/blake3/CMakeLists.txt b/lib/blake3/CMakeLists.txt new file mode 100644 index 00000000..e84d2a21 --- /dev/null +++ b/lib/blake3/CMakeLists.txt @@ -0,0 +1,54 @@ +if(CMAKE_ANDROID_ARM_NEON) + set ($BLAKE2_platform_sources "ext/blake3_neon.c") +elseif (MSVC OR MSYS OR MINGW) + set ($BLAKE2_platform_sources "ext/blake3_avx2_x86-64_windows_msvc.asm" "ext/blake3_avx512_x86-64_windows_msvc.asm" "ext/blake3_sse2_x86-64_windows_msvc.asm" "ext/blake3_sse41_x86-64_windows_msvc.asm") +elseif (WIN32) + set ($BLAKE2_platform_sources "ext/blake3_avx2_x86-64_windows_gnu.S" "ext/blake3_avx512_x86-64_windows_gnu.S" "ext/blake3_sse2_x86-64_windows_gnu.S" "ext/blake3_sse41_x86-64_windows_gnu.S") +else() + set ($BLAKE2_platform_sources "ext/blake3_avx2_x86-64_unix.S" "ext/blake3_avx512_x86-64_unix.S" "ext/blake3_sse2_x86-64_unix.S" "ext/blake3_sse41_x86-64_unix.S") +endif() + +add_library( + blake3 OBJECT + "longtail_blake3.c" + "longtail_blake3.h" + + "ext/blake3.c" + "ext/blake3.h" + "ext/blake3_dispatch.c" + "ext/blake3_impl.h" + "ext/blake3_portable.c" + ${BLAKE2_platform_sources} +) + +add_library( + blake3_sse2 OBJECT + "ext/blake3_sse2.c" +) + +add_library( + blake3_sse4_1 OBJECT + "ext/blake3_sse41.c" +) + +add_library( + blake3_avx2 OBJECT + "ext/blake3_avx2.c" +) + +add_library( + blake3_avx512 OBJECT + "ext/blake3_avx512.c" +) + +if(MSVC) + target_compile_options(blake3_sse2 INTERFACE /arch:SSE2) + target_compile_options(blake3_sse4_1 INTERFACE /arch:SSE4.1) + target_compile_options(blake3_avx2 INTERFACE /arch:AVX2) + target_compile_options(blake3_avx512 INTERFACE /arch:AVX512) +else() + target_compile_options(blake3_sse2 PUBLIC -msse2) + target_compile_options(blake3_sse4_1 PUBLIC -msse4.1) + target_compile_options(blake3_avx2 PUBLIC -mavx2) + target_compile_options(blake3_avx512 PUBLIC -mavx512vl -mavx512f) +endif(MSVC) diff --git a/lib/blockstorestorage/CMakeLists.txt b/lib/blockstorestorage/CMakeLists.txt new file mode 100644 index 00000000..2fb607cc --- /dev/null +++ b/lib/blockstorestorage/CMakeLists.txt @@ -0,0 +1 @@ +add_library(blockstorestorage OBJECT "longtail_blockstorestorage.c" "longtail_blockstorestorage.h") diff --git a/lib/brotli/CMakeLists.txt b/lib/brotli/CMakeLists.txt new file mode 100644 index 00000000..08c64e2d --- /dev/null +++ b/lib/brotli/CMakeLists.txt @@ -0,0 +1,94 @@ +add_library( + brotli OBJECT + "longtail_brotli.c" + "longtail_brotli.h" + + "ext/common/constants.c" + "ext/common/constants.h" + "ext/common/context.c" + "ext/common/context.h" + "ext/common/dictionary.c" + "ext/common/dictionary.h" + "ext/common/platform.c" + "ext/common/platform.h" + "ext/common/transform.c" + "ext/common/transform.h" + "ext/common/version.h" + + "ext/dec/bit_reader.h" + "ext/dec/bit_reader.c" + "ext/dec/decode.c" + "ext/dec/huffman.h" + "ext/dec/huffman.c" + "ext/dec/prefix.h" + "ext/dec/state.h" + "ext/dec/state.c" + + "ext/enc/backward_references.c" + "ext/enc/backward_references.h" + "ext/enc/backward_references_hq.c" + "ext/enc/backward_references_hq.h" + "ext/enc/backward_references_inc.h" + "ext/enc/bit_cost.c" + "ext/enc/bit_cost.h" + "ext/enc/bit_cost_inc.h" + "ext/enc/block_encoder_inc.h" + "ext/enc/block_splitter.c" + "ext/enc/block_splitter.h" + "ext/enc/block_splitter_inc.h" + "ext/enc/brotli_bit_stream.c" + "ext/enc/brotli_bit_stream.h" + "ext/enc/cluster.c" + "ext/enc/cluster.h" + "ext/enc/cluster_inc.h" + "ext/enc/command.c" + "ext/enc/command.h" + "ext/enc/compress_fragment.c" + "ext/enc/compress_fragment.h" + "ext/enc/compress_fragment_two_pass.c" + "ext/enc/compress_fragment_two_pass.h" + "ext/enc/dictionary_hash.c" + "ext/enc/dictionary_hash.h" + "ext/enc/encode.c" + "ext/enc/encoder_dict.c" + "ext/enc/encoder_dict.h" + "ext/enc/entropy_encode.c" + "ext/enc/entropy_encode.h" + "ext/enc/entropy_encode_static.h" + "ext/enc/fast_log.c" + "ext/enc/fast_log.h" + "ext/enc/find_match_length.h" + "ext/enc/hash.h" + "ext/enc/hash_composite_inc.h" + "ext/enc/hash_forgetful_chain_inc.h" + "ext/enc/hash_longest_match64_inc.h" + "ext/enc/hash_longest_match_inc.h" + "ext/enc/hash_longest_match_quickly_inc.h" + "ext/enc/hash_rolling_inc.h" + "ext/enc/hash_to_binary_tree_inc.h" + "ext/enc/histogram.c" + "ext/enc/histogram.h" + "ext/enc/histogram_inc.h" + "ext/enc/literal_cost.c" + "ext/enc/literal_cost.h" + "ext/enc/memory.c" + "ext/enc/memory.h" + "ext/enc/metablock.c" + "ext/enc/metablock.h" + "ext/enc/metablock_inc.h" + "ext/enc/params.h" + "ext/enc/prefix.h" + "ext/enc/quality.h" + "ext/enc/ringbuffer.h" + "ext/enc/static_dict.c" + "ext/enc/static_dict.h" + "ext/enc/static_dict_lut.h" + "ext/enc/utf8_util.c" + "ext/enc/utf8_util.h" + "ext/enc/write_bits.h" + + "ext/include/brotli/decode.h" + "ext/include/brotli/encode.h" + "ext/include/brotli/port.h" + "ext/include/brotli/types.h" +) diff --git a/lib/cacheblockstore/CMakeLists.txt b/lib/cacheblockstore/CMakeLists.txt new file mode 100644 index 00000000..f9bdabce --- /dev/null +++ b/lib/cacheblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(cacheblockstore OBJECT "longtail_cacheblockstore.c" "longtail_cacheblockstore.h") diff --git a/lib/compressblockstore/CMakeLists.txt b/lib/compressblockstore/CMakeLists.txt new file mode 100644 index 00000000..54146c95 --- /dev/null +++ b/lib/compressblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(compressblockstore OBJECT "longtail_compressblockstore.c" "longtail_compressblockstore.h") diff --git a/lib/compressionregistry/CMakeLists.txt b/lib/compressionregistry/CMakeLists.txt new file mode 100644 index 00000000..8007f405 --- /dev/null +++ b/lib/compressionregistry/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(compressionregistry_full + OBJECT + "longtail_compression_registry.c" + "longtail_compression_registry.h" + "longtail_full_compression_registry.c" + "longtail_full_compression_registry.h" +) + +add_library(compressionregistry_zstd + OBJECT + "longtail_compression_registry.c" + "longtail_compression_registry.h" + "longtail_zstd_compression_registry.c" + "longtail_zstd_compression_registry.h" +) diff --git a/lib/compressionregistry/longtail_full_compression_registry.c b/lib/compressionregistry/longtail_full_compression_registry.c index 4667a45f..77abf241 100644 --- a/lib/compressionregistry/longtail_full_compression_registry.c +++ b/lib/compressionregistry/longtail_full_compression_registry.c @@ -2,18 +2,18 @@ #include "longtail_compression_registry.h" -#include "../brotli/longtail_brotli.h" +//#include "../brotli/longtail_brotli.h" #include "../lz4/longtail_lz4.h" #include "../zstd/longtail_zstd.h" struct Longtail_CompressionRegistryAPI* Longtail_CreateFullCompressionRegistry() { - Longtail_CompressionRegistry_CreateForTypeFunc compression_create_api_funcs[3] = { - Longtail_CompressionRegistry_CreateForBrotli, + Longtail_CompressionRegistry_CreateForTypeFunc compression_create_api_funcs[2] = { + //Longtail_CompressionRegistry_CreateForBrotli, Longtail_CompressionRegistry_CreateForLZ4, Longtail_CompressionRegistry_CreateForZstd}; return Longtail_CreateDefaultCompressionRegistry( - 3, + 2, (const Longtail_CompressionRegistry_CreateForTypeFunc*)compression_create_api_funcs); } diff --git a/lib/filestorage/CMakeLists.txt b/lib/filestorage/CMakeLists.txt new file mode 100644 index 00000000..ed5f6a5a --- /dev/null +++ b/lib/filestorage/CMakeLists.txt @@ -0,0 +1 @@ +add_library(filestorage OBJECT "longtail_filestorage.c" "longtail_filestorage.h") diff --git a/lib/fsblockstore/CMakeLists.txt b/lib/fsblockstore/CMakeLists.txt new file mode 100644 index 00000000..e2e5a267 --- /dev/null +++ b/lib/fsblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(fsblockstore OBJECT "longtail_fsblockstore.c" "longtail_fsblockstore.h") diff --git a/lib/hashregistry/CMakeLists.txt b/lib/hashregistry/CMakeLists.txt new file mode 100644 index 00000000..5aa29185 --- /dev/null +++ b/lib/hashregistry/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library( + hashregistry OBJECT + "longtail_hash_registry.c" + "longtail_hash_registry.h" + + "longtail_blake3_hash_registry.c" + "longtail_blake3_hash_registry.h" + "longtail_full_hash_registry.c" + "longtail_full_hash_registry.h" +) diff --git a/lib/hpcdcchunker/CMakeLists.txt b/lib/hpcdcchunker/CMakeLists.txt new file mode 100644 index 00000000..9557f853 --- /dev/null +++ b/lib/hpcdcchunker/CMakeLists.txt @@ -0,0 +1 @@ +add_library(hpcdcchunker OBJECT "longtail_hpcdcchunker.c" "longtail_hpcdcchunker.h") diff --git a/lib/lrublockstore/CMakeLists.txt b/lib/lrublockstore/CMakeLists.txt new file mode 100644 index 00000000..1f5614d1 --- /dev/null +++ b/lib/lrublockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(lrublockstore OBJECT "longtail_lrublockstore.c" "longtail_lrublockstore.h") diff --git a/lib/lz4/CMakeLists.txt b/lib/lz4/CMakeLists.txt new file mode 100644 index 00000000..1069dae9 --- /dev/null +++ b/lib/lz4/CMakeLists.txt @@ -0,0 +1,14 @@ +add_library(lz4 OBJECT + "longtail_lz4.c" + "longtail_lz4.h" + + "ext/lz4.c" + "ext/lz4.h" + "ext/lz4frame.c" + "ext/lz4frame.h" + "ext/lz4frame_static.h" + "ext/lz4hc.c" + "ext/lz4hc.h" + "ext/xxhash.c" + "ext/xxhash.h" +) diff --git a/lib/memstorage/CMakeLists.txt b/lib/memstorage/CMakeLists.txt new file mode 100644 index 00000000..f3bee5b9 --- /dev/null +++ b/lib/memstorage/CMakeLists.txt @@ -0,0 +1 @@ +add_library(memstorage OBJECT "longtail_memstorage.c" "longtail_memstorage.h") diff --git a/lib/memtracer/CMakeLists.txt b/lib/memtracer/CMakeLists.txt new file mode 100644 index 00000000..e76455e2 --- /dev/null +++ b/lib/memtracer/CMakeLists.txt @@ -0,0 +1 @@ +add_library(memtracer OBJECT "longtail_memtracer.c" "longtail_memtracer.h") diff --git a/lib/meowhash/CMakeLists.txt b/lib/meowhash/CMakeLists.txt new file mode 100644 index 00000000..c4e92470 --- /dev/null +++ b/lib/meowhash/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library( + meowhash OBJECT + + "longtail_meowhash.c" + "longtail_meowhash.h" + + "ext/meow_hash_x64_aesni.h" +) + +if(MSVC) + target_compile_options(blake2 INTERFACE /arch:SSE2) + target_compile_options(blake2 PUBLIC -DHAVE_SSE2) +else() + target_compile_options(meowhash PUBLIC -maes -mavx2) +endif(MSVC) diff --git a/lib/ratelimitedprogress/CMakeLists.txt b/lib/ratelimitedprogress/CMakeLists.txt new file mode 100644 index 00000000..ce5d13ff --- /dev/null +++ b/lib/ratelimitedprogress/CMakeLists.txt @@ -0,0 +1 @@ +add_library(ratelimitedprogress OBJECT "longtail_ratelimitedprogress.c" "longtail_ratelimitedprogress.h") diff --git a/lib/shareblockstore/CMakeLists.txt b/lib/shareblockstore/CMakeLists.txt new file mode 100644 index 00000000..031f1142 --- /dev/null +++ b/lib/shareblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(shareblockstore OBJECT "longtail_shareblockstore.c" "longtail_shareblockstore.h") diff --git a/lib/zstd/CMakeLists.txt b/lib/zstd/CMakeLists.txt new file mode 100644 index 00000000..51527c4c --- /dev/null +++ b/lib/zstd/CMakeLists.txt @@ -0,0 +1,80 @@ +if(MSVC) +else() + set (zstd_platform_sources "ext/decompress/huf_decompress_amd64.S") +endif() + + +add_library( + zstd OBJECT + + "longtail_zstd.c" + "longtail_zstd.h" + + "ext/common/bits.h" + "ext/common/bitstream.h" + "ext/common/compiler.h" + "ext/common/cpu.h" + "ext/common/debug.c" + "ext/common/debug.h" + "ext/common/entropy_common.c" + "ext/common/error_private.c" + "ext/common/error_private.h" + "ext/common/fse.h" + "ext/common/fse_decompress.c" + "ext/common/huf.h" + "ext/common/mem.h" + "ext/common/pool.c" + "ext/common/pool.h" + "ext/common/portability_macros.h" + "ext/common/threading.c" + "ext/common/threading.h" + "ext/common/xxhash.c" + "ext/common/xxhash.h" + "ext/common/zstd_common.c" + "ext/common/zstd_deps.h" + "ext/common/zstd_internal.h" + "ext/common/zstd_trace.h" + + "ext/compress/clevels.h" + "ext/compress/fse_compress.c" + "ext/compress/hist.c" + "ext/compress/hist.h" + "ext/compress/huf_compress.c" + "ext/compress/zstdmt_compress.c" + "ext/compress/zstdmt_compress.h" + "ext/compress/zstd_compress.c" + "ext/compress/zstd_compress_internal.h" + "ext/compress/zstd_compress_literals.c" + "ext/compress/zstd_compress_literals.h" + "ext/compress/zstd_compress_sequences.c" + "ext/compress/zstd_compress_sequences.h" + "ext/compress/zstd_compress_superblock.c" + "ext/compress/zstd_compress_superblock.h" + "ext/compress/zstd_cwksp.h" + "ext/compress/zstd_double_fast.c" + "ext/compress/zstd_double_fast.h" + "ext/compress/zstd_fast.c" + "ext/compress/zstd_fast.h" + "ext/compress/zstd_lazy.c" + "ext/compress/zstd_lazy.h" + "ext/compress/zstd_ldm.c" + "ext/compress/zstd_ldm.h" + "ext/compress/zstd_ldm_geartab.h" + "ext/compress/zstd_opt.c" + "ext/compress/zstd_opt.h" + + "ext/decompress/huf_decompress.c" + "ext/decompress/huf_decompress_amd64.S" + "ext/decompress/zstd_ddict.c" + "ext/decompress/zstd_ddict.h" + "ext/decompress/zstd_decompress.c" + "ext/decompress/zstd_decompress_block.c" + "ext/decompress/zstd_decompress_block.h" + "ext/decompress/zstd_decompress_internal.h" + + "ext/zdict.h" + "ext/zstd.h" + "ext/zstd_errors.h" + + ${zstd_platform_sources} +) diff --git a/shared_lib/CMakeLists.txt b/shared_lib/CMakeLists.txt new file mode 100644 index 00000000..4e6bf7cb --- /dev/null +++ b/shared_lib/CMakeLists.txt @@ -0,0 +1,31 @@ +add_library(shared_lib + SHARED + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..3e59c1d3 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1 @@ +add_library(src STATIC "longtail.c" "longtail.h" "ext/stb_ds.c" "ext/stb_ds.h") diff --git a/static_lib/CMakeLists.txt b/static_lib/CMakeLists.txt new file mode 100644 index 00000000..7b58854e --- /dev/null +++ b/static_lib/CMakeLists.txt @@ -0,0 +1,31 @@ +add_library(static_lib + STATIC + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 00000000..1db477cf --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(test_base + "main.cpp" + "test.cpp" + + "ext/jc_test.h" +) + +target_link_libraries(test_base PRIVATE static_lib) diff --git a/test/test.cpp b/test/test.cpp index cea8c300..6f4ed466 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -6,7 +6,7 @@ #include "../lib/blake2/longtail_blake2.h" #include "../lib/blake3/longtail_blake3.h" #include "../lib/bikeshed/longtail_bikeshed.h" -#include "../lib/brotli/longtail_brotli.h" +//#include "../lib/brotli/longtail_brotli.h" #include "../lib/archiveblockstore/longtail_archiveblockstore.h" #include "../lib/atomiccancel/longtail_atomiccancel.h" #include "../lib/blockstorestorage/longtail_blockstorestorage.h" @@ -347,49 +347,49 @@ TEST(Longtail, Longtail_LZ4) Longtail_DisposeAPI(&compression_api->m_API); } -TEST(Longtail, Longtail_Brotli) -{ - Longtail_CompressionAPI* compression_api = Longtail_CreateBrotliCompressionAPI(); - ASSERT_NE((Longtail_CompressionAPI*)0, compression_api); - uint32_t compression_settings = Longtail_GetBrotliTextMaxQuality(); - - const char* raw_data = - "A very long file that should be able to be recreated" - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 2 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 3 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 4 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 5 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 6 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 7 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 8 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 9 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 10 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 11 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 12 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 13 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 14 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 15 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 16 in a long sequence of stuff." - "And in the end it is not the same, it is different, just because why not"; - - size_t data_len = strlen(raw_data) + 1; - size_t compressed_size = 0; - size_t max_compressed_size = compression_api->GetMaxCompressedSize(compression_api, compression_settings, data_len); - char* compressed_buffer = (char*)Longtail_Alloc(0, max_compressed_size); - ASSERT_NE((char*)0, compressed_buffer); - ASSERT_EQ(0, compression_api->Compress(compression_api, compression_settings, raw_data, compressed_buffer, data_len, max_compressed_size, &compressed_size)); - - char* decompressed_buffer = (char*)Longtail_Alloc(0, data_len); - ASSERT_NE((char*)0, decompressed_buffer); - size_t uncompressed_size; - ASSERT_EQ(0, compression_api->Decompress(compression_api, compressed_buffer, decompressed_buffer, compressed_size, data_len, &uncompressed_size)); - ASSERT_EQ(data_len, uncompressed_size); - ASSERT_STREQ(raw_data, decompressed_buffer); - Longtail_Free(decompressed_buffer); - Longtail_Free(compressed_buffer); - - Longtail_DisposeAPI(&compression_api->m_API); -} +//TEST(Longtail, Longtail_Brotli) +//{ +// Longtail_CompressionAPI* compression_api = Longtail_CreateBrotliCompressionAPI(); +// ASSERT_NE((Longtail_CompressionAPI*)0, compression_api); +// uint32_t compression_settings = Longtail_GetBrotliTextMaxQuality(); +// +// const char* raw_data = +// "A very long file that should be able to be recreated" +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 2 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 3 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 4 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 5 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 6 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 7 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 8 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 9 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 10 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 11 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 12 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 13 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 14 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 15 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 16 in a long sequence of stuff." +// "And in the end it is not the same, it is different, just because why not"; +// +// size_t data_len = strlen(raw_data) + 1; +// size_t compressed_size = 0; +// size_t max_compressed_size = compression_api->GetMaxCompressedSize(compression_api, compression_settings, data_len); +// char* compressed_buffer = (char*)Longtail_Alloc(0, max_compressed_size); +// ASSERT_NE((char*)0, compressed_buffer); +// ASSERT_EQ(0, compression_api->Compress(compression_api, compression_settings, raw_data, compressed_buffer, data_len, max_compressed_size, &compressed_size)); +// +// char* decompressed_buffer = (char*)Longtail_Alloc(0, data_len); +// ASSERT_NE((char*)0, decompressed_buffer); +// size_t uncompressed_size; +// ASSERT_EQ(0, compression_api->Decompress(compression_api, compressed_buffer, decompressed_buffer, compressed_size, data_len, &uncompressed_size)); +// ASSERT_EQ(data_len, uncompressed_size); +// ASSERT_STREQ(raw_data, decompressed_buffer); +// Longtail_Free(decompressed_buffer); +// Longtail_Free(compressed_buffer); +// +// Longtail_DisposeAPI(&compression_api->m_API); +//} TEST(Longtail, Longtail_ZStd) { @@ -1212,14 +1212,14 @@ static uint32_t* GetAssetTags(Longtail_StorageAPI* , const Longtail_FileInfos* f { uint32_t count = file_infos->m_Count; uint32_t* result = (uint32_t*)Longtail_Alloc(0, sizeof(uint32_t) * count); - const uint32_t compression_types[4] = { + const uint32_t compression_types[3] = { 0, - Longtail_GetBrotliGenericDefaultQuality(), +// Longtail_GetBrotliGenericDefaultQuality(), Longtail_GetLZ4DefaultQuality(), Longtail_GetZStdDefaultQuality()}; for (uint32_t i = 0; i < count; ++i) { - result[i] = compression_types[i % 4]; + result[i] = compression_types[i % 3]; } return result; }