eliott/ort
1
0
Fork 0
mirror of https://github.com/pykeio/ort synced 2026-04-25 16:34:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(sys): compress binaries with LZMA2

Browse Source 
LZMA2 is way slower than gzip, even in decompression, but this cuts the total size of all binaries by 40% (!!!)
This commit is contained in:
Carson M.
2025-12-06 03:40:02 -06:00
parent 4b3e2b28b8
commit ba3b5e10bb
5 changed files with 26 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ort-sys/Cargo.toml
View File

@@ -24,7 +24,7 @@ default = [ "std" ]
std = []
pkg-config = [ "dep:pkg-config" ]
training = []
download-binaries = [ "dep:ureq", "dep:flate2", "dep:hmac-sha256" ]
download-binaries = [ "dep:ureq", "dep:lzma-rust2", "dep:hmac-sha256" ]
copy-dylibs = []
disable-linking = []
@@ -58,7 +58,8 @@ __tls = []
[build-dependencies]
ureq = { version = "3", optional = true, default-features = false, features = [ "socks-proxy" ] }
flate2 = { version = "1.0", optional = true, default-features = false, features = [ "zlib-rs" ] }
# The `optimization` feature enables the use of unsafe code, which is unfortunate, but LZMA is so slow we kinda need all the speed we can get.
lzma-rust2 = { version = "0.15", optional = true, default-features = false, features = [ "std", "optimization" ] }
hmac-sha256 = { version = "1.1", optional = true }
pkg-config = { version = "0.3", optional = true }
glob = { version = "0.3", optional = true }

28
ort-sys/build/download/dist.txt
View File

@@ -1,14 +1,14 @@
cu12 x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc+cu12.tgz 3027fe6affe5ba364728ba8b36d27535dde65ed79f4ee296d4edca67186387c0
cu12 x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu+cu12.tgz 6312d22b066925506193d94406e03aef8d93a167dc4f3eb613afd76c50ac4ffd
wgpu x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc+wgpu.tgz f16600492b634a6152dc818058eb8ff5a06e55ed84c376197f0cac9012e654e8
nvrtx x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc+nvrtx.tgz 9ad74dff1e45ffba24ce6f83bd2a197f316ec08c10f428becad79d371e480554
wgpu x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu+wgpu.tgz 0436a165c00e5cc61dd7e32bfa3d21f0d615fa266239ac1b1288c343c97e5c04
none x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc.tgz a2346eaf7a72cc59530c2b8033d3201afbc609db69791b872c07578795e3769e
none aarch64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-pc-windows-msvc.tgz 9e6a5d5e5637a3793eeda653a0fbd8636b60ff64d6f826186d2cd67d8d01c977
none x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu.tgz 3bb3a486a0520ff58ade70b3292208d74c0cfdbc166af8530efd348fefefc280
none aarch64-linux-android https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-linux-android.tgz 7c81e64f1f49860927ff3e6cb326ee62d8c4e044644ab4cf7d2168dffdb77904
none aarch64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-unknown-linux-gnu.tgz 0ed96eb53eda80282649855f081a25cf8c129325fd894f83f6ae8c0b347d8577
nvrtx x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu+nvrtx.tgz 3dac85bc9a46fb7e6b9cbcb17e8ce7d6ce4646fa3b8118b13bb698f86a3ded2c
none aarch64-apple-ios https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-apple-ios.tgz 3c637c360df3f96954cbe71bc4e11f5ecf25bda66dbae356fcb47a1fc263082e
none aarch64-apple-ios-sim https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-apple-ios-sim.tgz 0589ca9c37d6316f98f7097b4e8678a5991c9d6b029b3adf69ef6f443be866d5
none aarch64-apple-darwin https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-apple-darwin.tgz dc8b33e642323acf0d523f0a0c957aa77fd0d679f267c2e0ade5ab61c491cd15
cu12 x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc+cu12.tar.lzma2 348c34e3f536d6db0be3d04c49ac4b8655b955e1dbd9cb2bb2464f29dd55b482
cu12 x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu+cu12.tar.lzma2 d3c01924b801c77ff17d300b24e6dcd46d378348a921a48d96f115f87074fbb1
wgpu x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc+wgpu.tar.lzma2 a9cc0046d631a7da72d76ded177047328cbe6102c5a8079a2c63ba6137c32322
nvrtx x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc+nvrtx.tar.lzma2 5708c4008e2acbb1f62cb0b1cdf02abca9314ea58105e524aadd96d0c06702ad
wgpu x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu+wgpu.tar.lzma2 c9be1443c2aab4688a2f52806b0dc7f421fda6d4b5ccb9522f4e0290a0205b0f
none x86_64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-pc-windows-msvc.tar.lzma2 a03948a02dec0c38485febb0e54ccb89af070d3adcbcb91e2e4fa5fec278943a
none aarch64-pc-windows-msvc https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-pc-windows-msvc.tar.lzma2 640096e762430fa0566fe022911376fc78ce88c48dc5c81a6fc973d8173e8658
none x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu.tar.lzma2 8c57d059aaaee407812a5698d6706c79e090ad69e1a14204309e802dcbbaa35f
none aarch64-linux-android https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-linux-android.tar.lzma2 37504d73103e903adf5a2e431b9912168da20bfb24241cd81eb0e72aad6f12b8
none aarch64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-unknown-linux-gnu.tar.lzma2 c25248c32d84f228b9d584b84b31e1577e4810d46beb5e304e9fa340c000176c
nvrtx x86_64-unknown-linux-gnu https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/x86_64-unknown-linux-gnu+nvrtx.tar.lzma2 1b122a579cafb9299e71ecec2b9ec86de25b4c180fe42e260df9738fed936ee7
none aarch64-apple-ios https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-apple-ios.tar.lzma2 e8ec605a27072a31086407d6eb2c6bcc5d4f567dc090f380000326d740ca72dc
none aarch64-apple-ios-sim https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-apple-ios-sim.tar.lzma2 8ae2dfc164b21d0a9f7b8ad046193eb20b4e9c198a21ba8dba0a71e962e15776
none aarch64-apple-darwin https://cdn.pyke.io/0/pyke:ort-rs/ms@1.23.2/aarch64-apple-darwin.tar.lzma2 0897a0e1b840566a97e5a49497b02cbc204be2d006815174b639bc99731840f9

2
ort-sys/build/download/extract.rs
View File

@@ -67,7 +67,7 @@ impl TarHeader {
}
pub fn extract_tgz<R: Read>(reader: &mut R, output: &Path) -> Result<(), Error> {
let mut tar = flate2::read::GzDecoder::new(reader);
let mut tar = lzma_rust2::Lzma2Reader::new(reader, 1 << 26, None);
let mut pad_container = [0; 512];
loop {

10
ort-sys/build/download/verify.rs
View File

@@ -1,4 +1,4 @@
use std::io::Read;
use std::io::{self, Read};
use hmac_sha256::Hash;
@@ -35,8 +35,12 @@ impl<R: Read> VerifyReader<R> {
Self { reader, state: Hash::new() }
}
pub fn finalize(self) -> ([u8; 32], R) {
(self.state.finalize(), self.reader)
pub fn finalize(mut self) -> io::Result<([u8; 32], R)> {
// For whatever reason, `lzma-rust2` leaves some data unread at the end of the stream. Make sure all of the stream is
// consumed so we get the correct hash.
io::copy(&mut self, &mut io::sink())?;
Ok((self.state.finalize(), self.reader))
}
}

2
ort-sys/build/main.rs
View File

@@ -138,7 +138,7 @@ cargo::error= | Alternatively, try a different backend like `ort-tract`; see htt
return;
}
let (calculated_hash, _) = verified_reader.finalize();
let (calculated_hash, _) = verified_reader.finalize().expect("Failed to finalize read");
if calculated_hash[..] != download::hex_str_to_bytes(dist.hash) {
println!(
r"cargo::error=⚠️ The hash of the file downloaded from `{}` does not match the expected hash. ⚠️