Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 1 addition & 22 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,31 +13,23 @@ permissions:

jobs:
build_release:
name: NIF ${{ matrix.nif }} - ${{ matrix.job.target }}${{ matrix.job.variant && format(' ({0})', matrix.job.variant) || '' }} (${{ matrix.job.os }})
name: NIF ${{ matrix.nif }} - ${{ matrix.job.target }} (${{ matrix.job.os }})
runs-on: ${{ matrix.job.os }}
strategy:
fail-fast: false
matrix:
nif: ["2.17", "2.16", "2.15"]
job:
# Non-x86 targets (no AVX2 variant needed)
- { target: arm-unknown-linux-gnueabihf , os: ubuntu-22.04 , use-cross: true }
- { target: aarch64-unknown-linux-gnu , os: ubuntu-22.04 , use-cross: true }
- { target: aarch64-unknown-linux-musl , os: ubuntu-22.04 , use-cross: true }
- { target: aarch64-apple-darwin , os: macos-14 }
- { target: riscv64gc-unknown-linux-gnu , os: ubuntu-22.04 , use-cross: true }
# x86_64 baseline (SSE2, 16-byte SIMD)
- { target: x86_64-apple-darwin , os: macos-15 }
- { target: x86_64-unknown-linux-gnu , os: ubuntu-22.04 }
- { target: x86_64-unknown-linux-musl , os: ubuntu-22.04 , use-cross: true }
- { target: x86_64-pc-windows-gnu , os: windows-2022 }
- { target: x86_64-pc-windows-msvc , os: windows-2022 }
# x86_64 AVX2 variant (32-byte SIMD for Haswell+ CPUs)
- { target: x86_64-apple-darwin , os: macos-15 , variant: avx2 }
- { target: x86_64-unknown-linux-gnu , os: ubuntu-22.04 , variant: avx2 }
- { target: x86_64-unknown-linux-musl , os: ubuntu-22.04 , use-cross: true, variant: avx2 }
- { target: x86_64-pc-windows-gnu , os: windows-2022 , variant: avx2 }
- { target: x86_64-pc-windows-msvc , os: windows-2022 , variant: avx2 }

env:
RUSTUP_TOOLCHAIN: nightly-2026-07-27
Expand Down Expand Up @@ -67,18 +59,6 @@ jobs:
toolchain: nightly-2026-07-27
targets: ${{ matrix.job.target }}

- name: Set RUSTFLAGS for AVX2 variant
if: matrix.job.variant == 'avx2'
shell: bash
run: echo "RUSTFLAGS=${RUSTFLAGS:+$RUSTFLAGS }-C target-cpu=x86-64-v3" >> $GITHUB_ENV

# When RUSTFLAGS env var is set, it overrides .cargo/config.toml rustflags.
# musl targets need -crt-static to produce cdylib (.so) output.
- name: Add musl cdylib flags
if: contains(matrix.job.target, 'musl') && env.RUSTFLAGS != ''
shell: bash
run: echo "RUSTFLAGS=${RUSTFLAGS} -C target-feature=-crt-static" >> $GITHUB_ENV

- name: Build the project
id: build-crate
uses: philss/rustler-precompiled-action@v1.1.4
Expand All @@ -89,7 +69,6 @@ jobs:
nif-version: ${{ matrix.nif }}
use-cross: ${{ matrix.job.use-cross }}
project-dir: "native/rustycsv"
variant: ${{ matrix.job.variant }}

- name: Artifact upload
uses: actions/upload-artifact@v4
Expand Down
7 changes: 5 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,11 @@ default CSV output.
this does not claim recovery from arbitrary native crashes.
- Moved all streaming resource locks onto dirty CPU schedulers.
- Pinned the Rust nightly used by source, CI, and release builds.
- Replaced the unconditional Windows AVX2 assumption with Windows' processor
feature check, retaining the optimized binary with safe baseline fallback.
- Removed build-host CPU probing and x86-64-v3 precompiled variants. All release
artifacts now use portable target baselines, so a CI or build machine cannot
select a binary that is incompatible with its deployment host.
- Kept the parser and encoder on safe portable SIMD after x86 benchmarks found
no consistent end-to-end NIF gain sufficient to justify runtime unsafe dispatch.

### Verification

Expand Down
8 changes: 4 additions & 4 deletions RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@
`git commit -m "Prepare vx.y.z"`
5. Push to main: `git push origin main`
6. Trigger NIF build: `gh workflow run release.yml --field version=x.y.z`
7. **Wait for ALL 45 builds to complete** (~5-10 min)
7. **Wait for ALL 30 builds to complete** (~5-10 min)
```bash
gh run watch <run-id>
```
8. Verify draft release has 45 assets:
8. Verify draft release has 30 assets:
```bash
gh release view vx.y.z --json assets --jq '.assets | length'
```
Expand All @@ -50,7 +50,7 @@

## Important Notes

- **Do NOT publish the draft release (step 8) until ALL 45 jobs complete and assets are attached**
- **Do NOT publish the draft release (step 8) until ALL 30 jobs complete and assets are attached**
- The workflow creates a draft release - each job attaches its asset to this draft
- Publishing too early causes a race condition where later jobs fail to attach their assets
- Step 7 verifies all assets are present before proceeding
Expand All @@ -65,7 +65,7 @@ gh run list --workflow=release.yml
gh run watch <run-id>

# Check draft release assets
gh release view vx.y.z --json assets --jq '.assets | length' # Should be 45
gh release view vx.y.z --json assets --jq '.assets | length' # Should be 30
gh release view vx.y.z --json assets --jq '.assets[].name'

# If something goes wrong, delete and retry
Expand Down
6 changes: 3 additions & 3 deletions docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ Unlike projects that wrap existing Rust crates (like the excellent `csv` crate),

### Validated Correctness

- **484 ExUnit tests plus 131 Rust tests** covering RFC 4180, industry test suites, edge cases, encodings, multi-byte separators/escapes, and headers-to-maps
- **494 ExUnit tests plus 5 properties and 125 Rust tests** covering RFC 4180, industry test suites, edge cases, encodings, multi-byte separators/escapes, and headers-to-maps
- **Cross-strategy validation** - All strategies produce identical output
- **NimbleCSV compatibility** - The upstream semantic test suites pass against RustyCSV

Expand Down Expand Up @@ -60,7 +60,7 @@ CSV.parse_string(huge_csv, strategy: :parallel)

# Dump back to CSV
CSV.dump_to_iodata([["a", "b"], ["1", "2"]])
#=> "a,b\n1,2\n"
#=> [["a,b\r\n"], ["1,2\r\n"]]
```

## NimbleCSV API Compatibility
Expand Down Expand Up @@ -188,7 +188,7 @@ All batch strategies share a single-pass SIMD structural scanner (`scan_structur

**How it works:**

1. Load 16-byte chunks (or 32-byte on AVX2) into SIMD registers
1. Load 16-byte chunks into portable SIMD registers
2. Compare against separator, quote, `\n`, and `\r` characters simultaneously
3. Use **prefix-XOR** on the quote bitmask to determine which positions are inside quoted regions — a cumulative XOR where bit *i* is set if there's an odd number of quotes before position *i*
4. Mask out quoted positions, then extract the remaining separator and newline positions into `Vec<u32>` arrays
Expand Down
41 changes: 0 additions & 41 deletions lib/rusty_csv/native.ex
Original file line number Diff line number Diff line change
Expand Up @@ -77,40 +77,6 @@ defmodule RustyCSV.Native do

version = Mix.Project.config()[:version]

avx2_detect = fn _config ->
try do
cond do
File.exists?("/proc/cpuinfo") ->
case File.read("/proc/cpuinfo") do
{:ok, info} -> String.contains?(info, "avx2")
_ -> false
end

match?({:win32, _}, :os.type()) ->
script =
~S/Add-Type -Namespace RustyCSV -Name CpuFeatures -MemberDefinition '[DllImport("kernel32.dll")] public static extern bool IsProcessorFeaturePresent(uint feature);'; [RustyCSV.CpuFeatures]::IsProcessorFeaturePresent(40)/

case System.cmd("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script],
stderr_to_stdout: true
) do
{result, 0} -> String.trim(result) == "True"
_ -> false
end

true ->
# macOS x86_64: check via sysctl
case System.cmd("sysctl", ["-n", "hw.optional.avx2_0"], stderr_to_stdout: true) do
{"1\n", 0} -> true
_ -> false
end
end
rescue
_ -> false
end
end

x86_64_variants = [avx2: avx2_detect]

use RustlerPrecompiled,
otp_app: :rusty_csv,
crate: "rustycsv",
Expand All @@ -122,13 +88,6 @@ defmodule RustyCSV.Native do
["aarch64-apple-darwin", "x86_64-apple-darwin"] ++
RustlerPrecompiled.Config.default_targets()
),
variants: %{
"x86_64-unknown-linux-gnu" => x86_64_variants,
"x86_64-apple-darwin" => x86_64_variants,
"x86_64-pc-windows-msvc" => x86_64_variants,
"x86_64-pc-windows-gnu" => x86_64_variants,
"x86_64-unknown-linux-musl" => x86_64_variants
},
version: version

# ==========================================================================
Expand Down
2 changes: 0 additions & 2 deletions native/rustycsv/src/core/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,3 @@ pub use simd_index::{
};
pub use simd_scanner::scan_structural;
pub use simd_scanner::CHUNK;
#[cfg(target_feature = "avx2")]
pub use simd_scanner::WIDE;
77 changes: 5 additions & 72 deletions native/rustycsv/src/core/simd_scanner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,16 @@
//! ## Bitmask types
//!
//! On current nightly, `Mask::to_bitmask()` returns u64 regardless of lane
//! count. We mask to the relevant bits (lower 16 for CHUNK=16, lower 32 for
//! WIDE=32) and operate on u64 uniformly.
//! count. We mask to the lower 16 bits for CHUNK=16 and operate on u64
//! uniformly.
//!
//! ## Optimization notes
//!
//! - Uses `to_bitmask()` + bit extraction (not `.any()` + break) because we
//! need ALL structural positions, not just the first one per chunk.
//! - Prefix-XOR for quote region detection: portable shift-and-xor cascade
//! for all targets, with CLMUL/PMULL fast paths on x86_64/aarch64.
//! - AVX2 wide path (32 bytes) processes first, then 16-byte remainder,
//! then scalar tail. Same pattern as RustyJSON's skip_plain_string_bytes.
//! on all targets.
//! - Processes 16-byte portable-SIMD chunks followed by a scalar tail.

use std::simd::prelude::*;

Expand All @@ -32,10 +31,6 @@ use super::simd_index::{InputTooLarge, RowEnd, StructuralIndex, Violation};
/// Baseline SIMD chunk size (128-bit).
pub const CHUNK: usize = 16;

/// Wide chunk size for AVX2 targets.
#[cfg(target_feature = "avx2")]
pub const WIDE: usize = 32;

// ---------------------------------------------------------------------------
// Prefix-XOR: compute cumulative XOR to determine quoted regions
// ---------------------------------------------------------------------------
Expand All @@ -44,8 +39,7 @@ pub const WIDE: usize = 32;
// prefix_xor(mask) produces a bitmask where bit i is set if position i is
// inside a quoted region (odd number of quotes before it).

/// Prefix-XOR via shift-and-xor cascade (works for 16 and 32 bits
/// within a u64, since upper bits are zero).
/// Prefix-XOR via shift-and-xor cascade for the lower 16 bits of a u64.
///
/// For these small bit widths the cascade is 6 dependent XOR+shift ops (~6 cycles),
/// comparable to a single CLMUL/PMULL instruction (~3-4 cycle latency + setup).
Expand Down Expand Up @@ -235,67 +229,6 @@ pub fn scan_structural(
let mut carry = QuoteCarry::at_start();
let mut violation: Option<Violation> = None;

// -----------------------------------------------------------------------
// AVX2 wide path: 32-byte chunks
// -----------------------------------------------------------------------
#[cfg(target_feature = "avx2")]
{
let esc_splat = Simd::<u8, WIDE>::splat(escape);
let lf_splat = Simd::<u8, WIDE>::splat(b'\n');
let cr_splat = Simd::<u8, WIDE>::splat(b'\r');

let sep_splats: Vec<Simd<u8, WIDE>> = separators
.iter()
.map(|&s| Simd::<u8, WIDE>::splat(s))
.collect();

const MASK_32: u64 = (1u64 << 32) - 1;

while pos + WIDE <= input.len() {
let chunk = Simd::<u8, WIDE>::from_slice(&input[pos..pos + WIDE]);
let base = pos as u32;

let esc_mask = chunk.simd_eq(esc_splat).to_bitmask() & MASK_32;

let raw_quoted = prefix_xor(esc_mask) & MASK_32;
let quoted = raw_quoted ^ (quote_carry.wrapping_neg() & MASK_32);

quote_carry ^= (esc_mask.count_ones() as u64) & 1;

let not_quoted = !quoted & MASK_32;

let mut sep_bits: u64 = 0;
for splat in &sep_splats {
sep_bits |= chunk.simd_eq(*splat).to_bitmask() & MASK_32;
}

let lf_raw = chunk.simd_eq(lf_splat).to_bitmask() & MASK_32;
let cr_raw = chunk.simd_eq(cr_splat).to_bitmask() & MASK_32;

let masks = ChunkMasks {
esc: esc_mask,
quoted,
sep: sep_bits,
lf: lf_raw,
cr: cr_raw,
};
let next_is_lf = matches!(input.get(pos + WIDE), Some(&b) if b == b'\n');
let bad = check_quoting(WIDE as u32, &masks, next_is_lf, &mut carry);
record_violation(bad, esc_mask, base, &mut violation);

extract_positions(sep_bits & not_quoted, base, &mut field_seps);
emit_row_ends(
input,
pos,
lf_raw & not_quoted,
cr_raw & not_quoted,
&mut row_ends,
);

pos += WIDE;
}
}

// -----------------------------------------------------------------------
// 16-byte chunks
// -----------------------------------------------------------------------
Expand Down
20 changes: 1 addition & 19 deletions native/rustycsv/src/strategy/encode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,12 @@
//! escapes").
//!
//! Scanning strategies:
//! SIMD: portable_simd 16/32-byte vectorized comparison (fastest)
//! SIMD: portable_simd 16-byte vectorized comparison (fastest)
//! General: byte-by-byte for multi-byte separator/escape patterns

use std::simd::prelude::*;

use crate::core::simd_scanner::CHUNK;
#[cfg(target_feature = "avx2")]
use crate::core::simd_scanner::WIDE;

// ==========================================================================
// Quoting: wrap field in escape chars, double internal escapes
Expand Down Expand Up @@ -121,22 +119,6 @@ pub fn field_needs_quoting(field: &[u8], reserved: &ReservedPatterns) -> bool {
let len = field.len();
let mut pos = 0;

// AVX2 wide path: 32 bytes at a time
#[cfg(target_feature = "avx2")]
if let Some((&first, rest)) = reserved.single_bytes.split_first() {
while pos + WIDE <= len {
let chunk = Simd::<u8, WIDE>::from_slice(&field[pos..pos + WIDE]);
let mut hits = chunk.simd_eq(Simd::splat(first));
for &byte in rest {
hits |= chunk.simd_eq(Simd::splat(byte));
}
if hits.any() {
return true;
}
pos += WIDE;
}
}

// 16-byte path
if let Some((&first, rest)) = reserved.single_bytes.split_first() {
while pos + CHUNK <= len {
Expand Down
Loading