Remove aligned block format and do all calculations in the standard r…

…aster coefficient format (#59)

* not working yet

* make sure we encode coefficients in the right order

* revert model changes

* changes

* fix idct

* optimize idct now that we dont have the weird aligned order

* update versions and build avx and non-avx2 versions

* always trigger on pull request

* add all build types

* add toolchains

* add all build types

* make compile under linux and wasm

* fix build

* fix rust build

* fix rust build

.github/workflows/rust.yml

@@ -11,14 +11,26 @@ env:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
 
     steps:
     - uses: actions/checkout@v3
-    - name: Build
-      run: cargo build --locked --verbose
+    - uses: dtolnay/rust-toolchain@stable
+      with:
+        toolchain: stable
+        targets: wasm32-wasi,aarch64-unknown-linux-musl,x86_64-pc-windows-msvc,x86_64-unknown-linux-gnu
+        components: rustfmt,clippy
+
+    - name: Build default target
+      run: cargo build --locked 
+    - name: Build wasm32-wasi
+      run: cargo build --locked --target wasm32-wasi --lib
+    - name: Build aarch64-unknown-linux-musl
+      run: cargo build --locked --target aarch64-unknown-linux-musl --lib
+    - name: Build x86_64-pc-windows-msvc
+      run: cargo build --locked --target x86_64-pc-windows-msvc --lib
     - name: Run tests
-      run: cargo test --locked --verbose
+      run: cargo test --locked
     - name: Check formatting
       run: cargo fmt --check
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lepton_jpeg"
-version = "0.3.4"
+version = "0.3.5"
 edition = "2021"
 authors = ["Kristof Roomp <[email protected]>"]
 
@@ -34,9 +34,11 @@ anyhow = { version="1.0", features = ["backtrace"]}
 wide = "0.7"
 log = "0.4"
 simple_logger ="4.0"
-cpu-time = "1.0"
 rayon = "1.10"
 
+[target.'cfg(windows)'.dependencies]
+cpu-time = "1.0"
+
 [dev-dependencies]
 rstest = "0.18"
 rand = "0.8"

azure-pipelines.yml

@@ -35,7 +35,16 @@ steps:
   inputs:
     mergeTestResults: true
 
-- script: 'cargo build --locked --release 2>&1'
+- script: |
+   set RUSTFLAGS=-Ccontrol-flow-guard -Ctarget-feature=+crt-static,+avx2,+lzcnt -Clink-args=/DYNAMICBASE/CETCOMPAT
+   cargo build --locked --release 2>&1
+   copy target\release\lepton_jpeg.dll target\release\lepton_jpeg_avx2.dll
+   copy target\release\lepton_jpeg.pdb target\release\lepton_jpeg_avx2.pdb
+   copy target\release\lepton_jpeg_util.exe target\release\lepton_jpeg_util_avx2.exe
+   copy target\release\lepton_jpeg_util.pdb target\release\lepton_jpeg_util_avx2.pdb
+   set RUSTFLAGS=-Ccontrol-flow-guard -Ctarget-feature=+crt-static -Clink-args=/DYNAMICBASE/CETCOMPAT
+   cargo build --locked --release 2>&1
+
   displayName: 'Build Release'
 
 - task: UseDotNet@2
@@ -51,7 +60,9 @@ steps:
       target\debug\lepton_jpeg.dll,
       target\debug\lepton_jpeg_util.exe,
       target\release\lepton_jpeg.dll,
+      target\release\lepton_jpeg_avx2.dll,
       target\release\lepton_jpeg_util.exe'
+      target\release\lepton_jpeg_util_avx2.exe'
     signConfigType: 'inlineSignParams'
     inlineOperation: |
       [

package/Lepton.Jpeg.Rust.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2011/08/nuspec.xsd">
   <metadata>
     <id>Lepton.Jpeg.Rust</id>
-    <version>0.3.4.3</version>
+    <version>0.3.5.0</version>
     <title>Lepton JPEG Compression Rust version binaries and libraries</title>
     <authors>kristofr</authors>
     <owners>kristofr</owners>
@@ -18,7 +18,11 @@
 
     <file src="..\target\release\lepton_jpeg_util.exe" target="exe\release\x64" />
     <file src="..\target\release\lepton_jpeg_util.pdb" target="exe\release\x64" />
+    <file src="..\target\release\lepton_jpeg_util_avx2.exe" target="exe\release\x64" />
+    <file src="..\target\release\lepton_jpeg_util_avx2.pdb" target="exe\release\x64" />
     <file src="..\target\release\lepton_jpeg.dll" target="lib\release\x64" />
     <file src="..\target\release\lepton_jpeg.pdb" target="lib\release\x64" />
+    <file src="..\target\release\lepton_jpeg_avx2.dll" target="lib\release\x64" />
+    <file src="..\target\release\lepton_jpeg_avx2.pdb" target="lib\release\x64" />
   </files>
 </package>

src/consts.rs

@@ -20,26 +20,18 @@ pub enum JPegType {
 
 pub const COLOR_CHANNEL_NUM_BLOCK_TYPES: usize = 3;
 
-pub const ALIGNED_BLOCK_INDEX_DC_INDEX: usize = 49;
-
-pub const ZIGZAG_TO_ALIGNED: [u8; 64] = [
-    49, 50, 57, 58, 0, 51, 52, 1, 2, 59, 60, 3, 4, 5, 53, 54, 6, 7, 8, 9, 61, 62, 10, 11, 12, 13,
-    14, 55, 56, 15, 16, 17, 18, 19, 20, 63, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
-    35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
-];
-
-pub const RASTER_TO_ALIGNED: [u8; 64] = [
-    49, 50, 51, 52, 53, 54, 55, 56, 57, 0, 1, 5, 6, 14, 15, 27, 58, 2, 4, 7, 13, 16, 26, 28, 59, 3,
-    8, 12, 17, 25, 29, 38, 60, 9, 11, 18, 24, 30, 37, 39, 61, 10, 19, 23, 31, 36, 40, 45, 62, 20,
-    22, 32, 35, 41, 44, 46, 63, 21, 33, 34, 42, 43, 47, 48,
-];
-
-pub const RASTER_TO_JPEG_ZIGZAG: [u8; 64] = [
+pub const RASTER_TO_ZIGZAG: [u8; 64] = [
     0, 1, 5, 6, 14, 15, 27, 28, 2, 4, 7, 13, 16, 26, 29, 42, 3, 8, 12, 17, 25, 30, 41, 43, 9, 11,
     18, 24, 31, 40, 44, 53, 10, 19, 23, 32, 39, 45, 52, 54, 20, 22, 33, 38, 46, 51, 55, 60, 21, 34,
     37, 47, 50, 56, 59, 61, 35, 36, 48, 49, 57, 58, 62, 63,
 ];
 
+pub const ZIGZAG_TO_RASTER: [u8; 64] = [
+    0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20,
+    13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59,
+    52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63,
+];
+
 pub const UNZIGZAG_49: [u8; 49] = [
     9, 10, 17, 25, 18, 11, 12, 19, 26, 33, 41, 34, 27, 20, 13, 14, 21, 28, 35, 42, 49, 57, 50, 43,
     36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62,
@@ -70,17 +62,6 @@ pub const FREQ_MAX: [u16; 64] = [
     1020, 1020, 1020, 854, 854, 838, 1020, 838, 1020, 838,
 ];
 
-pub const LOG_TABLE_256: [u8; 256] = [
-    0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-    5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
-    6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
-    6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
-    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
-    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
-    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
-    7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
-];
-
 pub const NON_ZERO_TO_BIN: [[u8; 50]; 49] = [
     [
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

src/main.rs

@@ -14,9 +14,9 @@ mod structs;
 
 use anyhow;
 use anyhow::Context;
-use cpu_time::ThreadTime;
 use helpers::err_exit_code;
 use lepton_error::{ExitCode, LeptonError};
+use lepton_jpeg::metrics::CpuTimeMeasure;
 use log::info;
 use simple_logger::SimpleLogger;
 use structs::lepton_format::read_jpeg;
@@ -204,7 +204,7 @@ fn main_with_result() -> anyhow::Result<()> {
 
     let mut current_iteration = 0;
     loop {
-        let thread_cpu = ThreadTime::now();
+        let thread_cpu = CpuTimeMeasure::new();
 
         if input_data[0] == 0xff && input_data[1] == 0xd8 {
             // the source is a JPEG file, so run the encoder and verify the results

src/metrics.rs

@@ -1,5 +1,38 @@
 use std::{collections::HashMap, time::Duration};
 
+#[cfg(windows)]
+use cpu_time::ThreadTime;
+
+/// platform independent threadtime measurement
+pub struct CpuTimeMeasure {
+    #[cfg(windows)]
+    start: ThreadTime,
+    #[cfg(not(windows))]
+    start: std::time::SystemTime,
+}
+
+impl CpuTimeMeasure {
+    pub fn new() -> Self {
+        Self {
+            #[cfg(windows)]
+            start: ThreadTime::now(),
+            #[cfg(not(windows))]
+            start: std::time::SystemTime::now(),
+        }
+    }
+
+    pub fn elapsed(&self) -> Duration {
+        #[cfg(windows)]
+        {
+            self.start.elapsed()
+        }
+        #[cfg(not(windows))]
+        {
+            self.start.elapsed().unwrap()
+        }
+    }
+}
+
 #[derive(Debug, PartialEq, Copy, Clone, Hash, Eq)]
 pub enum ModelSubComponent {
     Exp,

src/structs/block_based_image.rs

@@ -6,7 +6,7 @@
 
 use log::info;
 
-use crate::consts::{ALIGNED_BLOCK_INDEX_DC_INDEX, RASTER_TO_ALIGNED, ZIGZAG_TO_ALIGNED};
+use crate::consts::ZIGZAG_TO_RASTER;
 
 use super::{block_context::BlockContext, jpeg_header::JPegHeader};
 
@@ -146,11 +146,9 @@ impl BlockBasedImage {
         }
     }
 
-    pub fn set_block_data(&mut self, dpos: i32, block_data: &[i16; 64]) {
+    pub fn set_block_data(&mut self, dpos: i32, block_data: &AlignedBlock) {
         self.fill_up_to_dpos(dpos);
-        self.image[(dpos - self.dpos_offset) as usize] = AlignedBlock {
-            raw_data: *block_data,
-        };
+        *self.image[(dpos - self.dpos_offset) as usize].get_block_mut() = *block_data.get_block();
     }
 
     pub fn get_block(&self, dpos: i32) -> &AlignedBlock {
@@ -197,15 +195,11 @@ impl AlignedBlock {
     }
 
     pub fn get_dc(&self) -> i16 {
-        return self.raw_data[ALIGNED_BLOCK_INDEX_DC_INDEX];
+        return self.raw_data[0];
     }
 
     pub fn set_dc(&mut self, value: i16) {
-        self.raw_data[ALIGNED_BLOCK_INDEX_DC_INDEX] = value
-    }
-
-    pub fn set_coefficient_zigzag_block(block_data: &mut [i16; 64], index: u8, value: i16) {
-        block_data[usize::from(crate::consts::ZIGZAG_TO_ALIGNED[usize::from(index)])] = value;
+        self.raw_data[0] = value
     }
 
     /// gets underlying array of 64 coefficients (guaranteed to be 32-byte aligned)
@@ -230,10 +224,9 @@ impl AlignedBlock {
     }
 
     pub fn get_count_of_non_zeros_7x7(&self) -> u8 {
-        // with aligned (zigzag) arrangement, the 7x7 data is located in offsets 0..48
         let mut num_non_zeros7x7: u8 = 0;
-        for index in 0..49 {
-            if self.raw_data[index] != 0 {
+        for index in 9..64 {
+            if index & 0x7 != 0 && self.raw_data[index] != 0 {
                 num_non_zeros7x7 += 1;
             }
         }
@@ -250,14 +243,10 @@ impl AlignedBlock {
     }
 
     pub fn set_coefficient_zigzag(&mut self, index: usize, v: i16) {
-        self.raw_data[usize::from(ZIGZAG_TO_ALIGNED[index])] = v;
-    }
-
-    pub fn get_coefficient_raster(&self, index: usize) -> i16 {
-        return self.raw_data[usize::from(RASTER_TO_ALIGNED[index])];
+        self.raw_data[usize::from(ZIGZAG_TO_RASTER[index])] = v;
     }
 
     pub fn get_coefficient_zigzag(&self, index: usize) -> i16 {
-        return self.raw_data[usize::from(ZIGZAG_TO_ALIGNED[index])];
+        return self.raw_data[usize::from(ZIGZAG_TO_RASTER[index])];
     }
 }