Scheduled upgrade + native zlib (#23)

* upgrade dependencies * start splitting * split * native artifact naming * typo * artifacts etc. * typo * envs * cmake requirement? * build * create fat native * and publish * deps * path for fats * more unit tests * wrong artifact name * i gave up on zlib-ng * adding native zlib compression support * fix exports * wrong size reporting * bug in managed compression test * comment
aloneguid · Sep 24, 2024 · aa2380c · aa2380c
1 parent 9fef92b
commit aa2380c
Show file tree

Hide file tree

Showing 16 changed files with 358 additions and 177 deletions.
diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml
@@ -1,11 +1,12 @@
 name: 'Full Flow'
 
 env:
-  VERSION: 1.5.2
-  # PACKAGE_SUFFIX: '-pre.1'
-  PACKAGE_SUFFIX: ''
+  VERSION: 1.6.0
+  PACKAGE_SUFFIX: '-pre.1'
+  # PACKAGE_SUFFIX: ''
   ASM_VERSION: 1.0.0
   BUILD_TYPE: Release
+  VCPKG_HASH: 0f88ecb8528605f91980b90a2c5bad88e3cb565f
   # Indicates the location of the vcpkg as a Git submodule of the project repository.
   #VCPKG_ROOT: ${{ github.workspace }}/vcpkg
   # Tells vcpkg where binary packages are stored.
@@ -28,95 +29,76 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
-        arch: [x64, arm64]
-        #include:
-        #  - os: windows-latest
-        #    arch: x86
-        exclude:
-          # no build agent for Windows ARM
-          - os: windows-latest
+        include:
+          - os: ubuntu-20.04
+            arch: x64
+            vcpkg_triplet: x64-linux
+            vcpkg_config: RelWithDebInfo
+            dotnet_rid: linux-x64
+          - os: ubuntu-20.04
             arch: arm64
-          # no more x64 agent available on GitHub Actions
-          - os: macos-latest
+            vcpkg_triplet: arm64-linux
+            vcpkg_config: RelWithDebInfo
+            dotnet_rid: linux-arm64
+          - os: windows-latest
             arch: x64
+            vcpkg_triplet: x64-windows-static
+            vcpkg_config: Release
+            dotnet_rid: win-x64
+          - os: macos-latest
+            arch: arm64  
+            vcpkg_triplet: arm64-osx
+            vcpkg_config: RelWithDebInfo
+            dotnet_rid: osx-arm64
+
       fail-fast: false
     name: 'build: ${{ matrix.os }} (${{ matrix.arch }})'
 
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
-    - name: Windows Env
-      if: matrix.os == 'windows-latest'
-      run: |
-        echo "VCPKG_DEFAULT_VCPKG_TRIPLET=${{ matrix.arch }}-windows-static" >> $env:GITHUB_ENV
-        echo "VCPKG_TRIPLET=${{ matrix.arch }}-windows-static" >> $env:GITHUB_ENV
-        echo "VCPKG_CONFIG=Release" >> $env:GITHUB_ENV
-
-    - name: Linux Env
-      if: matrix.os == 'ubuntu-20.04'
-      run: |
-        echo "VCPKG_DEFAULT_VCPKG_TRIPLET=${{ matrix.arch }}-linux" >> $GITHUB_ENV
-        echo "VCPKG_TRIPLET=${{ matrix.arch }}-linux" >> $GITHUB_ENV
-        echo "VCPKG_CONFIG=RelWithDebInfo" >> $GITHUB_ENV
-
-    - name: MacOS Env
-      if: matrix.os == 'macos-latest'
-      run: |
-        echo "VCPKG_DEFAULT_VCPKG_TRIPLET=${{ matrix.arch }}-osx" >> $GITHUB_ENV
-        echo "VCPKG_TRIPLET=${{ matrix.arch }}-osx" >> $GITHUB_ENV
-        echo "VCPKG_CONFIG=RelWithDebInfo" >> $GITHUB_ENV
-
-    - name: Windows x64
-      if: runner.os == 'Windows' && matrix.arch == 'x64'
-      run: |
-        echo "DOTNET_RID=win-x64" >> $env:GITHUB_ENV
-
-    - name: Windows x32
-      if: runner.os == 'Windows' && matrix.arch == 'x86'
+    - name: env (windows)
+      if: runner.os == 'Windows'
       run: |
-        echo "VCPKG_DEFAULT_VCPKG_TRIPLET=${{ matrix.arch }}-windows" >> $env:GITHUB_ENV
-        echo "VCPKG_TRIPLET=${{ matrix.arch }}-windows" >> $env:GITHUB_ENV
-        echo "DOTNET_RID=win-x86" >> $env:GITHUB_ENV
-        echo "CMAKE_OPTIONS=-A Win32" >> $env:GITHUB_ENV
+        echo "VCPKG_DEFAULT_VCPKG_TRIPLET=${{ matrix.vcpkg_triplet }}" >> $env:GITHUB_ENV
+        echo "VCPKG_TRIPLET=${{ matrix.vcpkg_triplet }}" >> $env:GITHUB_ENV
+        echo "VCPKG_CONFIG=${{ matrix.vcpkg_config }}" >> $env:GITHUB_ENV
+        echo "DOTNET_RID=${{ matrix.dotnet_rid }}" >> $env:GITHUB_ENV
 
-    - name: Linux x64
-      if: runner.os == 'Linux' && matrix.arch == 'x64'
+    - name: env
+      if: runner.os == 'Linux' || runner.os == 'macOS'
       run: |
-        echo "DOTNET_RID=linux-x64" >> $GITHUB_ENV
+        echo "VCPKG_DEFAULT_VCPKG_TRIPLET=${{ matrix.vcpkg_triplet }}" >> $GITHUB_ENV
+        echo "VCPKG_TRIPLET=${{ matrix.vcpkg_triplet }}" >> $GITHUB_ENV
+        echo "VCPKG_CONFIG=${{ matrix.vcpkg_config }}" >> $GITHUB_ENV
+        echo "DOTNET_RID=${{ matrix.dotnet_rid }}" >> $GITHUB_ENV
 
     - name: Linux ARM
       if: runner.os == 'Linux' && matrix.arch == 'arm64'
       run: |
         sudo apt-get update
         sudo apt-get --yes install g++-aarch64-linux-gnu pkg-config-aarch64-linux-gnu
         echo "CMAKE_OPTIONS=-D CMAKE_SYSTEM_PROCESSOR=aarch64 -D CMAKE_C_COMPILER=$(which aarch64-linux-gnu-gcc) -D CMAKE_CXX_COMPILER=$(which aarch64-linux-gnu-g++)" >> $GITHUB_ENV
-        echo "DOTNET_RID=linux-arm64" >> $GITHUB_ENV
-
-    - name: OSX x64
-      if: runner.os == 'macOS' && matrix.arch == 'x64'
-      run: |
-        echo "DOTNET_RID=osx-x64" >> $GITHUB_ENV
+        echo "SUPPRESS_TEST=1" >> $GITHUB_ENV
 
     - name: OSX M1
       if: runner.os == 'macOS' && matrix.arch == 'arm64'
       run: |
         echo "CMAKE_OPTIONS=-D CMAKE_OSX_ARCHITECTURES=arm64" >> $GITHUB_ENV
-        echo "DOTNET_RID=osx-arm64" >> $GITHUB_ENV
 
     - uses: friendlyanon/setup-vcpkg@v1
       # seems like the absense of commit hash expects vcpkg submodule, i don't want that
       with:
-        committish: 261dd6831673d3b07e5118261f7e161a21d1a759 
+        committish: ${{ env.VCPKG_HASH }} 
         # set to false to clear any cache in case of build errors
         cache: false
 
-    - name: show cmake config
-      run: cmake -h
-      working-directory: native
+    # - name: show cmake config
+    #   run: cmake -h
+    #   working-directory: native
 
     - name: configure
       run: cmake -B build -S . -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -D "CMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -D "VCPKG_TARGET_TRIPLET=${{ env.VCPKG_TRIPLET }}" ${{ env.CMAKE_OPTIONS }}
@@ -130,7 +112,31 @@ jobs:
       run: ls -R
       working-directory: native
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
+      name: Collect Artifacts
+      with:
+        name: native-${{ matrix.dotnet_rid }}
+        path: |
+          native/ubin/**/*
+        if-no-files-found: error
+
+  # create a unified fat native binary
+  make-fat-native:
+    runs-on: ubuntu-latest
+    needs: build-native
+    steps:
+
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: native-*
+        path: native/ubin/
+        merge-multiple: true
+
+    - name: debug
+      run: ls -R
+      working-directory: .
+
+    - uses: actions/upload-artifact@v4
       name: Collect Artifacts
       with:
         name: native
@@ -139,10 +145,18 @@ jobs:
 
   test-managed:
     runs-on: ${{ matrix.os }}
-    needs: build-native
+    needs: make-fat-native
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, windows-2022, macos-latest]
+        include:
+          - os: ubuntu-20.04
+            dotnet_rid: linux-x64
+          - os: ubuntu-22.04
+            dotnet_rid: linux-x64
+          - os: windows-2022
+            dotnet_rid: win-x64 
+          - os: macos-latest
+            dotnet_rid: osx-arm64
       fail-fast: false
     name: 'test: ${{ matrix.os }}'
 
@@ -152,11 +166,11 @@ jobs:
 
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: native
         path: native/ubin/
@@ -183,15 +197,15 @@ jobs:
       run: dotnet test managed/IronCompress.sln -c release /p:Version=${{ env.VERSION }}${{ env.PACKAGE_SUFFIX }} /p:FileVersion=$VERSION /p:AssemblyVersion=$ASM_VERSION
 
   build-managed:
-    needs: build-native
+    needs: make-fat-native
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v4
       with:
         name: native
         path: native/ubin/
@@ -210,7 +224,7 @@ jobs:
     - name: Build
       run: dotnet build managed/IronCompress.sln -c release /p:Version=${{ env.VERSION }}${{ env.PACKAGE_SUFFIX }} /p:FileVersion=$VERSION /p:AssemblyVersion=$ASM_VERSION
 
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
       name: Collect Artifacts
       with:
         name: managed
@@ -224,7 +238,7 @@ jobs:
     environment: nuget
     steps:
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         with:
           name: managed
           path: pub/managed/

diff --git a/docs/README.md b/docs/README.md
@@ -23,7 +23,9 @@ And following architectures:
 | **Zstd**   | ✅                                               |      | ✅       | ⛔     | ⛔       | x32              |
 |            |                                                 |      | ✅       | ✅     | ⛔       | x64              |
 |            |                                                 |      | ⛔       | ✅     | ✅       | ARM64            |
-| **Gzip**   | ✅<br /><sup>*part of .NET*</sup>                |      |         |       |         |                  |
+| **Gzip**   | ✅<br /><sup>*part of .NET*</sup>                |      | ⛔       | ⛔     | ⛔       | x32              |
+|            |                                                 |      | ✅       | ✅     | ✅       | x64              |
+|            |                                                 |      | ⛔       | ✅     | ✅       | ARM64            |
 | **Brotli** | ✅<br /><sup>*except on .NET Standard 2.0*</sup> |      | ✅       | ⛔     | ⛔       | x32              |
 |            |                                                 |      | ✅       | ✅     | ⛔       | x64              |
 |            |                                                 |      | ⛔       | ✅     | ✅       | ARM64            |
@@ -73,7 +75,7 @@ using (IronCompressResult uncompressed = iron.Decompress(Codec.Snappy, compresse
 
 As with compression, this returns `Result` with decompressed data. It's worth nothing one important difference - decompression needs the length of output buffer specified as third parameter (`input.Length`). Although some decompressors can guess uncompressed length more or less correct, the others won't know it beforehand. In reality this problem is solved by using a framing format that adds metadata about resulting length, however many compression formats do not define that and consider compressed packets to be implementation specific.
 
-You will need more or less recent C++ compiler, `CMake` and `.NET SDK 6` to build the code.
+You will need more or less recent C++ compiler, `CMake` and `.NET SDK 8` to build the code.
 
 
 ## Building

diff --git a/managed/IronCompress.Benchmarks/IronCompress.Benchmarks.csproj b/managed/IronCompress.Benchmarks/IronCompress.Benchmarks.csproj
@@ -8,7 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/managed/IronCompress.Test/IronCompress.Test.csproj b/managed/IronCompress.Test/IronCompress.Test.csproj
@@ -7,9 +7,9 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
-        <PackageReference Include="xunit" Version="2.8.0" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.0">
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+        <PackageReference Include="xunit" Version="2.9.1" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>

diff --git a/managed/IronCompress.Test/IronTest.cs b/managed/IronCompress.Test/IronTest.cs
@@ -55,7 +55,7 @@ public void EncodeDecodeManaged(Codec codec) {
         _rnd.NextBytes(input);
 
         using(IronCompressResult compressed = iron.Compress(codec, input.AsSpan())) {
-            using(IronCompressResult uncompressed = _iron.Decompress(codec, compressed, input.Length)) {
+            using(IronCompressResult uncompressed = iron.Decompress(codec, compressed, input.Length)) {
                 Assert.Equal(input, uncompressed.AsSpan().ToArray());
             }
         }

diff --git a/managed/IronCompress/Iron.cs b/managed/IronCompress/Iron.cs
@@ -21,7 +21,7 @@ public static bool IsNativeLibraryAvailable {
                 // cache the results, as it won't change during the lifetime of the app
                 if(_isNativeLibraryAvailable == null) {
                     try {
-                        Native.ping();
+                        Native.iron_ping();
                         _isNativeLibraryAvailable = true;
 
                     } catch(DllNotFoundException) {
@@ -44,7 +44,8 @@ static bool SupportsManaged(Codec c) {
         }
 
         static bool SupportsNative(Codec c) {
-            return IsNativeLibraryAvailable && c != Codec.Gzip;
+            //return IsNativeLibraryAvailable && c != Codec.Gzip;
+            return true;
         }
 
         /// <summary>
@@ -131,7 +132,7 @@ private IronCompressResult NativeCompressOrDecompress(
                 fixed(byte* inputPtr = input) {
                     // get output buffer size into "len"
                     if(outputLength == null) {
-                        bool ok = Native.compress(
+                        bool ok = Native.iron_compress(
                            compressOrDecompress,
                            (int)codec, inputPtr, input.Length, null, &len, level);
                         if(!ok) {
@@ -147,7 +148,7 @@ private IronCompressResult NativeCompressOrDecompress(
 
                     fixed(byte* outputPtr = output) {
                         try {
-                            bool ok = Native.compress(
+                            bool ok = Native.iron_compress(
                                compressOrDecompress,
                                (int)codec, inputPtr, input.Length, outputPtr, &len, level);
 

diff --git a/managed/IronCompress/Native.cs b/managed/IronCompress/Native.cs
@@ -38,7 +38,7 @@ private static IntPtr DllImportResolver(string libraryName, Assembly assembly, D
 #endif
 
         [DllImport(LibName)]
-        internal static extern unsafe bool compress(bool compress,
+        internal static extern unsafe bool iron_compress(bool compress,
             int codec,
             byte* inputBuffer,
             int inputBufferSize,
@@ -47,9 +47,6 @@ internal static extern unsafe bool compress(bool compress,
             int compressionLevel);
 
         [DllImport(LibName)]
-        internal static extern bool ping();
-
-        [DllImport(LibName)]
-        internal static extern bool is_supported(int codec);
+        internal static extern bool iron_ping();
     }
 }