eliott/ort
1
0
Fork 0
mirror of https://github.com/pykeio/ort synced 2026-04-25 16:34:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

docs: nuke mintlify

Browse Source
This commit is contained in:
Carson M.
2024-06-08 13:19:34 -05:00
parent 1d89f822d8
commit b00c9341bc
26 changed files with 3571 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
docs/mint.json
View File

@@ -1,102 +0,0 @@
{
"$schema": "https://mintlify.com/schema.json",
"name": "ort",
"logo": {
"dark": "/assets/banner.png",
"light": "/assets/banner.png"
},
"favicon": "/assets/icon.png",
"colors": {
"primary": "#F74C00",
"light": "#F74C00",
"background": {
"light": "#FFFFFF",
"dark": "#000000"
},
"dark": "#F74C00",
"anchors": {
"from": "#F74C00",
"to": "#eb8e65"
}
},
"tabs": [
{
"name": "API Reference",
"url": "https://docs.rs/ort/2.0.0-rc.2/ort/"
}
],
"anchors": [
{
"name": "Sponsor",
"icon": "hand-holding-heart",
"url": "https://opencollective.com/pyke-osai"
},
{
"name": "Crates.io",
"icon": "rust",
"url": "https://crates.io/crates/ort"
},
{
"name": "GitHub",
"icon": "github",
"url": "https://github.com/pykeio/ort"
},
{
"name": "Discord",
"icon": "discord",
"url": "https://discord.gg/uQtsNu2xMa"
}
],
"navigation": [
{
"group": "Get Started",
"pages": [
"introduction"
]
},
{
"group": "Setup",
"pages": [
"setup/platforms",
"setup/webassembly",
"setup/linking",
"setup/cargo-features"
]
},
{
"group": "Fundamentals",
"pages": [
"fundamentals/environment",
"fundamentals/session",
"fundamentals/value"
]
},
{
"group": "Performance",
"pages": [
"perf/execution-providers",
"perf/io-binding"
]
},
{
"group": "Troubleshooting",
"pages": [
"troubleshooting/precision",
"troubleshooting/performance",
"troubleshooting/compiling"
]
},
{
"group": "Migration & versioning",
"pages": [
"migrating/version-mapping",
"migrating/v2"
]
}
],
"footerSocials": {
"website": "https://pyke.io/",
"github": "https://github.com/pykeio/ort",
"discord": "https://discord.gg/uQtsNu2xMa"
}
}

5
docs/next-env.d.ts vendored Normal file
View File

@@ -0,0 +1,5 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
// NOTE: This file should not be edited
// see https://nextjs.org/docs/basic-features/typescript for more information.

11
docs/next.config.mjs Normal file
View File

@@ -0,0 +1,11 @@
import nextra from 'nextra';
export default nextra({
theme: 'nextra-theme-docs',
themeConfig: './theme.config.jsx'
})({
output: 'export',
images: {
unoptimized: true
}
});

23
docs/package.json Normal file
View File

@@ -0,0 +1,23 @@
{
"private": true,
"name": "ort-docs",
"version": "0.0.0",
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start"
},
"dependencies": {
"next": "^14.2.3",
"nextra": "^2.13.4",
"nextra-theme-docs": "^2.13.4",
"react": "^18.3.1",
"react-dom": "^18.3.1"
},
"devDependencies": {
"@types/node": "20.14.2",
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"typescript": "^5.4.5"
}
}

5
docs/pages/_app.mdx Normal file
View File

@@ -0,0 +1,5 @@
import font from 'next/font/google';
export default function App({ Component, pageProps }) {
return <Component {...pageProps} />;
}

37
docs/pages/_meta.json Normal file
View File

@@ -0,0 +1,37 @@
{
"-- Links": {
"type": "separator",
"title": "Links"
},
"link-oc": {
"title": "Sponsor ↗",
"href": "https://opencollective.com/pyke-osai",
"newWindow": true
},
"link-api": {
"title": "API Reference ↗",
"href": "https://docs.rs/ort/2.0.0-rc.2/ort"
},
"link-crates": {
"title": "Crates.io ↗",
"href": "https://crates.io/crates/ort",
"newWindow": true
},
"-- Docs": {
"type": "separator",
"title": "Docs"
},
"index": "Introduction",
"setup": {
"title": "Setup"
},
"perf": {
"title": "Performance"
},
"troubleshooting": {
"title": "Troubleshooting"
},
"migrating": {
"title": "Migration & versioning"
}
}

89
docs/introduction.mdx → docs/pages/index.mdx
View File

@@ -2,14 +2,17 @@
title: Introduction
---
import Image from 'next/image';
import { Callout, Card, Cards, Steps } from 'nextra/components';
<div style={{ display: 'flex', flexDirection: 'column', alignItems: 'center' }}>
<img src="/assets/trend-banner.png" style={{ maxHeight: '360px' }} />
<p style={{ fontSize: '1.5rem', textAlign: 'center', marginTop: 0 }}><code>ort</code> is an open-source Rust binding for <a href="https://onnxruntime.ai/">ONNX Runtime</a>.</p>
</div>
<Warning>
<Callout type='warning'>
These docs are for the latest alpha version of `ort`, `2.0.0-rc.2`. This version is production-ready (just not API stable) and we recommend new & existing projects use it.
</Warning>
</Callout>
`ort` makes it easy to deploy your machine learning models to production via [ONNX Runtime](https://onnxruntime.ai/), a hardware-accelerated inference engine. With `ort` + ONNX Runtime, you can run almost any ML model (including ResNet, YOLOv8, BERT, LLaMA) on almost any hardware, often far faster than PyTorch, and with the added bonus of Rust's efficiency.
@@ -29,52 +32,54 @@ Converting a neural network to a graph representation like ONNX opens the door t
# Getting started
<Steps>
<Step title="Add ort to your Cargo.toml">
If you have a [supported platform](/setup/platforms) (and you probably do), installing `ort` couldn't be any simpler! Just add it to your Cargo dependencies:
```toml
[dependencies]
ort = "2.0.0-rc.2"
```
</Step>
<Step title="Convert your model">
Your model will need to be converted to an ONNX graph before you can use it.
- The awesome folks at Hugging Face have [a guide](https://huggingface.co/docs/transformers/serialization) to export 🤗 Transformers models to ONNX with 🤗 Optimum.
- For any PyTorch model: [`torch.onnx`](https://pytorch.org/docs/stable/onnx.html)
- For `scikit-learn` models: [`sklearn-onnx`](https://onnx.ai/sklearn-onnx/)
- For TensorFlow, Keras, TFlite, & TensorFlow.js: [`tf2onnx`](https://github.com/onnx/tensorflow-onnx)
- For PaddlePaddle: [`Paddle2ONNX`](https://github.com/PaddlePaddle/Paddle2ONNX)
</Step>
<Step title="Load your model">
Once you've got a model, load it via `ort` by creating a [`Session`](/fundamentals/session):
```rust
use ort::{GraphOptimizationLevel, Session};
### Add ort to your Cargo.toml
If you have a [supported platform](/setup/platforms) (and you probably do), installing `ort` couldn't be any simpler! Just add it to your Cargo dependencies:
```toml
[dependencies]
ort = "2.0.0-rc.2"
```
let model = Session::builder()?
.with_optimization_level(GraphOptimizationLevel::Level3)?
.with_intra_threads(4)?
.commit_from_file("yolov8m.onnx")?;
```
</Step>
<Step title="Perform inference">
Preprocess your inputs, then `run()` the session to perform inference.
### Convert your model
Your model will need to be converted to an ONNX graph before you can use it.
- The awesome folks at Hugging Face have [a guide](https://huggingface.co/docs/transformers/serialization) to export 🤗 Transformers models to ONNX with 🤗 Optimum.
- For any PyTorch model: [`torch.onnx`](https://pytorch.org/docs/stable/onnx.html)
- For `scikit-learn` models: [`sklearn-onnx`](https://onnx.ai/sklearn-onnx/)
- For TensorFlow, Keras, TFlite, & TensorFlow.js: [`tf2onnx`](https://github.com/onnx/tensorflow-onnx)
- For PaddlePaddle: [`Paddle2ONNX`](https://github.com/PaddlePaddle/Paddle2ONNX)
```rust
let outputs = model.run(ort::inputs!["image" => image]?)?;
let predictions = outputs["output0"].try_extract_tensor::<f32>()?;
...
```
### Load your model
Once you've got a model, load it via `ort` by creating a [`Session`](/fundamentals/session):
```rust
use ort::{GraphOptimizationLevel, Session};
let model = Session::builder()?
.with_optimization_level(GraphOptimizationLevel::Level3)?
.with_intra_threads(4)?
.commit_from_file("yolov8m.onnx")?;
```
### Perform inference
Preprocess your inputs, then `run()` the session to perform inference.
```rust
let outputs = model.run(ort::inputs!["image" => image]?)?;
let predictions = outputs["output0"].try_extract_tensor::<f32>()?;
...
```
<Callout type='info'>There are some more useful examples [in the `ort` repo](https://github.com/pykeio/ort/tree/main/examples)!</Callout>
<Note>There are some more useful examples [in the `ort` repo](https://github.com/pykeio/ort/tree/main/examples)!</Note>
</Step>
</Steps>
# Next steps
<Steps>
<Step title="Unlock more performance with EPs">
Use [execution providers](/perf/execution-providers) to enable hardware acceleration in your app and unlock the full power of your GPU or NPU.
</Step>
<Step title="Show off your project!">
We'd love to see what you've made with `ort`! Show off your project in [GitHub Discussions](https://github.com/pykeio/ort/discussions/categories/show-and-tell) or on our [Discord](https://discord.gg/uQtsNu2xMa).
</Step>
### Unlock more performance with EPs
Use [execution providers](/perf/execution-providers) to enable hardware acceleration in your app and unlock the full power of your GPU or NPU.
### Show off your project!
We'd love to see what you've made with `ort`! Show off your project in [GitHub Discussions](https://github.com/pykeio/ort/discussions/categories/show-and-tell) or on our [Discord](https://discord.gg/uQtsNu2xMa).
</Steps>

0
docs/migrating/opsets.mdx → docs/pages/migrating/opsets.mdx
View File

7
docs/migrating/v2.mdx → docs/pages/migrating/v2.mdx
View File

@@ -141,13 +141,6 @@ let noise_pred = unet.run(ort::inputs![
]?)?;
```
You can also supply `ort::inputs!` your `IoBinding` by specifying `bind =`:
```rust
let binding = model.create_binding()?;
...
let outputs = model.run(ort::inputs![bind = binding]?)?;
```
### Tensor creation no longer requires the session's allocator
In previous versions, `Value::from_array` took an allocator parameter. The allocator was only used because the string data contained in string tensors had to be cloned into ONNX Runtime-managed memory. However, 99% of users only ever use primitive tensors, so the extra parameter served little purpose. The new `Tensor::from_array` function now takes only an array, and the logic for converting string arrays has been moved to a new function, `DynTensor::from_string_array`.

0
docs/migrating/version-mapping.mdx → docs/pages/migrating/version-mapping.mdx
View File

34
docs/perf/execution-providers.mdx → docs/pages/perf/execution-providers.mdx
View File

@@ -3,6 +3,8 @@ title: Execution providers
description: Learn how to enable execution providers to leverage hardware acceleration.
---
import { Callout, Tabs } from 'nextra/components';
Execution providers (EPs) enable ONNX Runtime to execute ONNX graphs with hardware acceleration. If you have specialized hardware like a GPU or NPU, execution providers can provide a massive performance boost to your `ort` applications. For more information on the intricacies of execution providers, see the [ONNX Runtime docs](https://onnxruntime.ai/docs/execution-providers/).
ONNX Runtime must be compiled with support for each execution provider. pyke provides precompiled binaries for some of the most common EPs, so you won't need to compile ONNX Runtime from source. Below is a table showing available EPs, their support in `ort`, and their binary availability status.
@@ -28,12 +30,12 @@ ONNX Runtime must be compiled with support for each execution provider. pyke pro
| Microsoft Azure | ❌ | ❌ | ❓ |
| Rockchip RKNPU | ❌ | ❌ | ❓ |
<Note>
<Callout type='default'>
Some EPs supported by ONNX Runtime are not supported by `ort` due to a lack of hardware for testing. If your preferred EP is missing support and you've got the hardware, please [open an issue](https://github.com/pykeio/ort/issues/new)!
</Note>
</Callout>
## Registering execution providers
<Info>
<Callout type='info'>
To use an execution provider with `ort`, you'll need to enable its respective Cargo feature, e.g. the `cuda` feature to use CUDA, or the `coreml` feature to use CoreML.
```toml Cargo.toml
@@ -42,7 +44,7 @@ ONNX Runtime must be compiled with support for each execution provider. pyke pro
```
See [Cargo features](/setup/cargo-features) for the full list of features.
</Info>
</Callout>
In order to configure sessions to use certain execution providers, you must **register** them when creating an environment or session. You can do this via the `SessionBuilder::with_execution_providers` method. For example, to register the CUDA execution provider for a session:
@@ -167,9 +169,9 @@ fn main() -> anyhow::Result<()> {
}
```
<Warning>
<Callout type='warning'>
`ort::init` must come before you create any sessions, otherwise the configuration will not take effect!
</Warning>
</Callout>
Sessions configured with their own execution providers will *extend* the execution provider defaults, rather than overriding them.
@@ -181,32 +183,32 @@ If it seems like the execution provider is not registering properly, or you are
### CoreML
Statically linking to CoreML (the default behavior when using downloaded binaries + the `coreml` Cargo feature) requires an additional Rust flag in order to link properly. You'll need to provide the flag `-C link-arg=-fapple-link-rtlib` to `rustc`. You can do this via an entry in [`.cargo/config.toml`](https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure), in a build script, or in an environment variable.
<Tabs>
<Tab title="Cargo config">
<Tabs items={['Cargo config', 'Build script', 'Environment variable']}>
<Tabs.Tab title="Cargo config">
See [Configuration: Hierarchical structure](https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure) for more information on where the configuration file can be placed.
```toml .cargo/config.toml
```toml filename=".cargo/config.toml" copy
[target.aarch64-apple-darwin]
rustflags = ["-Clink-arg=-fapple-link-rtlib"]
[target.x86_64-apple-darwin]
rustflags = ["-Clink-arg=-fapple-link-rtlib"]
```
</Tab>
<Tab title="Build script">
</Tabs.Tab>
<Tabs.Tab title="Build script">
Add the following to the `build.rs` script of any **binary** crate that uses `ort`.
```rust build.rs
```rust filename="build.rs" copy
fn main() {
println!("cargo:rustc-link-arg=-fapple-link-rtlib");
}
```
Library crates do not need this flag, and the usage of it in a library crate will not transitively apply to any binary crates dependent on it.
</Tab>
<Tab title="Environment variable">
```shell
</Tabs.Tab>
<Tabs.Tab title="Environment variable">
```shell copy
$ RUSTFLAGS="-Clink-arg=-fapple-link-rtlib" cargo build
```
</Tab>
</Tabs.Tab>
</Tabs>

0
docs/perf/io-binding.mdx → docs/pages/perf/io-binding.mdx
View File

0
docs/setup/cargo-features.mdx → docs/pages/setup/cargo-features.mdx
View File

109
docs/pages/setup/linking.mdx Normal file
View File

@@ -0,0 +1,109 @@
---
title: Linking
description: Here's how `ort` links to ONNX Runtime, and how to configure its behavior.
---
import { Callout, Tabs, Steps } from 'nextra/components';
`ort` provides its own builds of ONNX Runtime to make your experience as painless as possible, but in some cases, you'll want to use a custom build of ONNX Runtime with `ort`. Luckily, we make this very easy by handling all of the linking configuration automagically. Just point `ort` to the output of ONNX Runtime's build pipeline and it'll Just Work™.
## Static linking
Most ONNX Runtime compile configurations will support static linking - just run `build.sh` without the `--build_shared_lib` argument. You should prefer static linking if your execution providers support it, as it avoids many issues and follows de facto Rust practices. If you compile both static libraries and dynamic libraries, `ort` will prefer linking to the static libraries.
To direct `ort` to your statically built binaries, use the `ORT_LIB_LOCATION` environment variable when running `cargo build`. Point it to the location where the static libraries (`.a`/`.lib` files) are compiled to. This will typically be `onnxruntime/build/<os>`. For example:
```shell
$ ORT_LIB_LOCATION=~/onnxruntime/build/Linux cargo build
```
For iOS (or for other platforms if you are compiling multiple profiles at once), you'll need to manually specify the profile with the `ORT_LIB_PROFILE` environment variable. If not specified, `ort` will prefer `Release` over `RelWithDebInfo` over `MinSizeRel` over `Debug`.
## Dynamic linking
Some execution providers unfortunately only support dynamic linking. Dynamic linking doesn't play well with the Rust ecosystem, though `ort` tries to alleviate the pain as much as possible.
When it comes to dynamic linking, there are two options: `load-dynamic`, or standard compile-time dynamic linking. We recommend `load-dynamic` as it gives more control and is often far less troublesome to work with.
### Runtime loading with `load-dynamic`
The `load-dynamic` Cargo feature solves a few of the issues with dynamic linking by **loading the library at runtime** rather than **linking at compile time**. This means that the path to the ONNX Runtime library can be configured at runtime, and the executable will not just completely fail to start if the binary couldn't be found.
To use `load-dynamic`:
<Steps>
#### Enable the feature in Cargo.toml
```toml filename="Cargo.toml"
[dependencies]
ort = { version = "2", features = [ "load-dynamic" ] }
```
### Point ort to the dylib
<Tabs items={['Programmatically', 'Via shell']}>
<Tabs.Tab title="Programmatically">
```rust main.rs
fn main() -> anyhow::Result<()> {
// Find our custom ONNX Runtime dylib path somehow
// (i.e. resolving it from the root of our program's install folder)
let dylib_path = crate::internal::find_onnxruntime_dylib()?;
// The path should point to the `libonnxruntime` binary, which looks like:
// - on Unix: /etc/.../libonnxruntime.so
// - on Windows: C:\Program Files\...\onnxruntime.dll
// Initialize ort with the path to the dylib. This **must** be called before any usage of `ort`!
// `init_from` returns an `EnvironmentBuilder` which you can use to further configure the environment
// before `.commit()`ing; see the Environment docs for more information on what you can configure.
ort::init_from(dylib_path).commit()?;
Ok(())
}
```
</Tabs.Tab>
<Tabs.Tab title="Via shell">
Set the `ORT_DYLIB_PATH` environment variable to the path to `libonnxruntime.so`/`onnxruntime.dll`.
```shell
$ ORT_DYLIB_PATH=../onnxruntime-build/linux-x64/libonnxruntime.so ./mirai
```
</Tabs.Tab>
</Tabs>
</Steps>
<Callout type='info'>`ORT_DYLIB_PATH` is relative to the executable. Cargo examples and tests are compiled to a different directory than binary crates: `target/<profile>/examples` and `target/<profile>/deps` respectively. Keep this in mind if you're going to use relative paths.</Callout>
### Compile-time dynamic linking
For compile-time dynamic linking, you'll need to configure your environment in the exact same way as if you were [statically linking](#static-linking).
Note that the dylibs then have to be placed in a certain location for them to be found by the executable. For Windows, this is either somewhere on the `PATH`, or in the same folder as the executable. On macOS and Linux, they have to be placed somewhere in the `LD_LIBRARY_PATH`, or you can use rpath to configure the executable to search for dylibs in its parent folder. We've had the least issues with rpath, but YMMV.
To configure rpath, you'll need to:
<Steps>
#### Enable rpath in Cargo.toml
```toml filename="Cargo.toml" copy
[profile.dev]
rpath = true
[profile.release]
rpath = true
# do this for any other profiles
```
### Configure the path in the linker args in .cargo/config.toml to be relative to the executable
<Tabs items={['Linux', 'macOS']}>
<Tabs.Tab title="Linux">
```toml filename="~/.cargo/config.toml" copy
[target.x86_64-unknown-linux-gnu]
rustflags = [ "-Clink-args=-Wl,-rpath,\\$ORIGIN" ]
# do this for any other Linux targets as well
```
</Tabs.Tab>
<Tabs.Tab title="macOS">
```toml filename="~/.cargo/config.toml" copy
[target.x86_64-apple-darwin]
rustflags = [ "-Clink-args=-Wl,-rpath,@loader_path" ]
# do this for any other macOS targets as well
```
</Tabs.Tab>
</Tabs>
</Steps>

18
docs/setup/platforms.mdx → docs/pages/setup/platforms.mdx
View File

@@ -3,6 +3,8 @@ title: Platform support
description: ONNX Runtime, and by extension `ort`, supports a wide variety of platforms. For most desktop users, pre-built binaries are available, so setting up `ort` is as simple as adding it to your `Cargo.toml`!
---
import { Callout } from 'nextra/components';
Here are the supported platforms and binary availability status, as of v2.0.0-rc.2.
* 🟢 - Supported. Dynamic & static binaries provided by pyke.
@@ -19,14 +21,18 @@ Here are the supported platforms and binary availability status, as of v2.0.0-rc
| **Android** | ❌ | ❌ | ⭕ | ⭕ | ❌ |
| **Web** | ❌ | ❌ | ❌ | ❌ | 🔷¶ |
<sup>\* Recent version of Windows 10/11 required for pyke binaries.</sup><br />
<sup>† glibc ≥ 2.31 (Ubuntu ≥ 20.04) required for pyke binaries.</sup><br />
<sup>‡ glibc ≥ 2.35 (Ubuntu ≥ 22.04) required for pyke binaries.</sup><br />
<sup>§ macOS ≥ 10.15 required.</sup><br />
<sup>¶ WASM supports a limited subset of ONNX Runtime features. For more info, see [the docs on WebAssembly support](/setup/webassembly).</sup>
<div style={{ opacity: 0.5, display: 'flex', flexDirection: 'column', fontSize: '0.75rem' }}>
<p>\* Recent version of Windows 10/11 required for pyke binaries.</p>
<p>† glibc ≥ 2.31 (Ubuntu ≥ 20.04) required for pyke binaries.</p>
<p>‡ glibc ≥ 2.35 (Ubuntu ≥ 22.04) required for pyke binaries.</p>
<p>§ macOS ≥ 10.15 required.</p>
<p>¶ WASM supports a limited subset of ONNX Runtime features. For more info, see [the docs on WebAssembly support](/setup/webassembly).</p>
</div>
If your platform is marked as 🟢 or 🔷, you're in luck! Almost no setup will be required to get `ort` up and running.
For platforms marked as ⭕, you'll need to [compile ONNX Runtime from source](https://onnxruntime.ai/docs/build/) and then [link `ort` to your custom binaries](/setup/linking) (but don't worry, we made this setup as simple as possible!)
<Note>Certain execution providers may not have binaries available. You can check EP binary support in the [Execution providers](/perf/execution-providers) documentation.</Note>
<Callout type='info'>
Certain execution providers may not have binaries available. You can check EP binary support in the [Execution providers](/perf/execution-providers) documentation.
</Callout>

8
docs/setup/webassembly.mdx → docs/pages/setup/webassembly.mdx
View File

@@ -5,19 +5,13 @@ description: Deploy ONNX models to the web
WebAssembly support in `ort` is currently experimental. If you experience any issues using `ort` in WebAssembly, please [open an issue](https://github.com/pykeio/ort/issues/new).
Development of WASM support is done in a separate branch for now, so you'll have to add `ort` as a Git dependency:
```toml Cargo.toml
[dependencies]
ort = { git = "https://github.com/pykeio/ort.git", branch = "wasm32-unknown-unknown" }
```
By nature, some features of ONNX Runtime are not available in the web. These include:
- **Support for `.onnx` models.** You instead need to [convert `.onnx` models to the `.ort` format](https://onnxruntime.ai/docs/performance/model-optimizations/ort-format-models.html).
- **Runtime graph optimizations**, aka `SessionBuilder::with_optimization_level`. You can statically optimize the graph using the `.ort` conversion tool, though.
- **Loading models with `commit_from_file`/`commit_from_url`.** You can create models from a slice of bytes in memory with `SessionBuilder::commit_from_memory` or `SessionBuilder::commit_from_memory_directly`.
Additionally, you'll need to call `ort::wasm::initialize()` at the earliest possible point in your code, before you use any `ort` APIs:
```rust main.rs
```rust filename="main.rs" copy
use ort::Session;
static MODEL_BYTES: &[u8] = include_bytes!("../model.ort");

0
docs/troubleshooting/compiling.mdx → docs/pages/troubleshooting/compiling.mdx
View File

81
docs/troubleshooting/performance.mdx → docs/pages/troubleshooting/performance.mdx
View File

@@ -2,53 +2,56 @@
title: 'Troubleshoot: Performance'
---
import { Callout, Tabs, Steps } from 'nextra/components';
## Execution providers don't seem to register
`ort` is designed to fail gracefully when an execution provider is not available. It logs failure events through [`tracing`](https://crates.io/crates/tracing), thus you'll need a library that subscribes to `tracing` events to see the logs. The simplest way to do this is to use [`tracing-subscriber`](https://crates.io/crates/tracing-subscriber).
<Steps>
<Step title="Add tracing-subscriber to your dependencies">
```toml Cargo.toml
[dependencies]
tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }
### Add tracing-subscriber to your dependencies
```toml Cargo.toml
[dependencies]
tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }
```
### Initialize the subscriber in the main function
```rust main.rs
fn main() {
tracing_subscriber::fmt::init();
}
```
### Show debug messages from ort
Set the environment variable `RUST_LOG` to `ort=debug` to see all debug messages from `ort`.
<Tabs items={['Windows (PowerShell)', 'Windows (Command Prompt)', 'Linux', 'macOS']}>
<Tabs.Tab title="Windows (PowerShell)">
```powershell
$env:RUST_LOG = 'ort=debug';
cargo run
```
</Step>
<Step title="Initialize the subscriber in the main function">
```rust main.rs
fn main() {
tracing_subscriber::fmt::init();
}
</Tabs.Tab>
<Tabs.Tab title="Windows (Command Prompt)">
```cmd
set RUST_LOG=ort=debug
cargo run
```
</Step>
<Step title="Show debug messages from ort">
Set the environment variable `RUST_LOG` to `ort=debug` to see all debug messages from `ort`.
<Tabs>
<Tab title="Windows (PowerShell)">
```powershell
$env:RUST_LOG = 'ort=debug';
cargo run
```
</Tab>
<Tab title="Windows (Command Prompt)">
```cmd
set RUST_LOG=ort=debug
cargo run
```
</Tab>
<Tab title="Linux">
```shell
RUST_LOG="ort=debug" cargo run
```
</Tab>
<Tab title="macOS">
```shell
RUST_LOG="ort=debug" cargo run
```
</Tab>
</Tabs>
</Step>
</Tabs.Tab>
<Tabs.Tab title="Linux">
```shell
RUST_LOG="ort=debug" cargo run
```
</Tabs.Tab>
<Tabs.Tab title="macOS">
```shell
RUST_LOG="ort=debug" cargo run
```
</Tabs.Tab>
</Tabs>
</Steps>
<Note>You can also detect EP regsitration failures programmatically. See [Execution providers: Fallback behavior](/perf/execution-providers#fallback-behavior) for more info.</Note>
<Callout type='info'>You can also detect EP regsitration failures programmatically. See [Execution providers: Fallback behavior](/perf/execution-providers#fallback-behavior) for more info.</Callout>
## Inference is slower than expected
There are a few things you could try to improve performance:

3200
docs/pnpm-lock.yaml generated Normal file
View File

File diff suppressed because it is too large Load Diff

0
docs/assets/banner.png → docs/public/assets/banner.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 68 KiB

After

Width:  |  Height:  |  Size: 68 KiB

0
docs/assets/icon.png → docs/public/assets/icon.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 64 KiB

After

Width:  |  Height:  |  Size: 64 KiB

0
docs/assets/sample-onnx-graph.png → docs/public/assets/sample-onnx-graph.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 126 KiB

After

Width:  |  Height:  |  Size: 126 KiB

0
docs/assets/trend-banner.png → docs/public/assets/trend-banner.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 157 KiB

After

Width:  |  Height:  |  Size: 157 KiB

106
docs/setup/linking.mdx
View File

@@ -1,106 +0,0 @@
---
title: Linking
description: Here's how `ort` links to ONNX Runtime, and how to configure its behavior.
---
In some cases, you'll want to use a custom build of ONNX Runtime with `ort`. Luckily, we make this very easy by handling all of the linking configuration automagically. Just point `ort` to the output of ONNX Runtime's build pipeline and it'll Just Work™.
## Static linking
Most ONNX Runtime compile configurations will support static linking - just run `build.sh` without the `--build_shared_lib` argument. You should prefer static linking if your execution providers support it, as it avoids many issues and follows de facto Rust practices. If you compile both static libraries and dynamic libraries, `ort` will prefer linking to the static libraries.
To direct `ort` to your statically built binaries, use the `ORT_LIB_LOCATION` environment variable when running `cargo build`. Point it to the location where the static libraries (`.a`/`.lib` files) are compiled to. This will typically be `onnxruntime/build/<os>`. For example:
```shell
$ ORT_LIB_LOCATION=~/onnxruntime/build/Linux cargo build
```
For iOS (or for other platforms if you are compiling multiple profiles at once), you'll need to manually specify the profile with the `ORT_LIB_PROFILE` environment variable. If not specified, `ort` will prefer `Release` over `RelWithDebInfo` over `MinSizeRel` over `Debug`.
## Dynamic linking
Some execution providers unfortunately only support dynamic linking. Dynamic linking doesn't play well with the Rust ecosystem, though `ort` tries to alleviate the pain as much as possible.
When it comes to dynamic linking, there are two options: `load-dynamic`, or standard compile-time dynamic linking. We recommend `load-dynamic` as it gives more control and is often far less troublesome to work with.
### Runtime loading with `load-dynamic`
The `load-dynamic` Cargo feature solves a few of the issues with dynamic linking by **loading the library at runtime** rather than **linking at compile time**. This means that the path to the ONNX Runtime library can be configured at runtime, and the executable will not just completely fail to start if the binary couldn't be found.
To use `load-dynamic`:
<Steps>
<Step title="Enable the feature in Cargo.toml">
```toml Cargo.toml
[dependencies]
ort = { version = "2", features = [ "load-dynamic" ] }
```
</Step>
<Step title="Point ort to the dylib">
<Tabs>
<Tab title="Programmatically">
```rust main.rs
fn main() -> anyhow::Result<()> {
// Find our custom ONNX Runtime dylib path somehow
// (i.e. resolving it from the root of our program's install folder)
let dylib_path = crate::internal::find_onnxruntime_dylib()?;
// The path should point to the `libonnxruntime` binary, which looks like:
// - on Unix: /etc/.../libonnxruntime.so
// - on Windows: C:\Program Files\...\onnxruntime.dll
// Initialize ort with the path to the dylib. This **must** be called before any usage of `ort`!
// `init_from` returns an `EnvironmentBuilder` which you can use to further configure the environment
// before `.commit()`ing; see the Environment docs for more information on what you can configure.
ort::init_from(dylib_path).commit()?;
Ok(())
}
```
</Tab>
<Tab title="Via shell">
Set the `ORT_DYLIB_PATH` environment variable to the path to `libonnxruntime.so`/`onnxruntime.dll`.
```shell
$ ORT_DYLIB_PATH=../onnxruntime-build/linux-x64/libonnxruntime.so ./mirai
```
</Tab>
</Tabs>
</Step>
</Steps>
<Note>`ORT_DYLIB_PATH` is relative to the executable. Cargo examples and tests are compiled to a different directory than binary crates: `target/<profile>/examples` and `target/<profile>/deps` respectively. Keep this in mind if you're going to use relative paths.</Note>
### Compile-time dynamic linking
For compile-time dynamic linking, you'll need to configure your environment in the exact same way as if you were [statically linking](#static-linking).
Note that the dylibs then have to be placed in a certain location for them to be found by the executable. For Windows, this is either somewhere on the `PATH`, or in the same folder as the executable. On macOS and Linux, they have to be placed somewhere in the `LD_LIBRARY_PATH`, or you can use rpath to configure the executable to search for dylibs in its parent folder. We've had the least issues with rpath, but YMMV.
To configure rpath, you'll need to:
<Steps>
<Step title="Enable rpath in Cargo.toml">
```toml
[profile.dev]
rpath = true
[profile.release]
rpath = true
# do this for any other profiles
```
</Step>
<Step title="Configure the path in the linker args in .cargo/config.toml to be relative to the executable">
<Tabs>
<Tab title="Linux">
```toml
[target.x86_64-unknown-linux-gnu]
rustflags = [ "-Clink-args=-Wl,-rpath,\\$ORIGIN" ]
# do this for any other Linux targets as well
```
</Tab>
<Tab title="macOS">
```toml
[target.x86_64-apple-darwin]
rustflags = [ "-Clink-args=-Wl,-rpath,@loader_path" ]
# do this for any other macOS targets as well
```
</Tab>
</Tabs>
</Step>
</Steps>

33
docs/theme.config.jsx Normal file
View File

@@ -0,0 +1,33 @@
import Image from 'next/image';
/** @type {import('nextra-theme-docs').DocsThemeConfig} */
const config = {
project: {
link: 'https://github.com/pykeio/ort'
},
chat: {
link: 'https://discord.gg/uQtsNu2xMa'
},
docsRepositoryBase: 'https://github.com/pykeio/ort/blob/main/docs',
useNextSeoProps() {
return {
titleTemplate: '%s | ort'
}
},
logo: <img src="/assets/banner.png" style={{ height: '34px' }} />,
darkMode: true,
nextThemes: {
defaultTheme: 'system'
},
footer: {
text: <div>
<p>made with 💜 by <a target="_blank" href="https://pyke.io/" style={{ textDecoration: 'underline', textDecorationColor: '#05c485' }}><span style={{ color: '#2ba9f6' }}>py</span><span style={{ color: '#00c875' }}>ke</span></a> • <a target="_blank" href="https://opencollective.com/pyke-osai" style={{ textDecoration: 'underline' }}>sponsor</a></p>
</div>
},
primaryHue: 20,
primarySaturation: 100,
toc: {
float: true
}
};
export default config;

28
docs/tsconfig.json Normal file
View File

@@ -0,0 +1,28 @@
{
"compilerOptions": {
"lib": [
"dom",
"dom.iterable",
"esnext"
],
"allowJs": true,
"skipLibCheck": true,
"strict": false,
"noEmit": true,
"incremental": true,
"module": "esnext",
"esModuleInterop": true,
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve"
},
"include": [
"next-env.d.ts",
"**/*.ts",
"**/*.tsx"
, "pages/_app.mdx" ],
"exclude": [
"node_modules"
]
}