diff --git a/polars-dataframe/Cargo.toml b/polars-dataframe/Cargo.toml index 05153bc46b..f829c671fc 100644 --- a/polars-dataframe/Cargo.toml +++ b/polars-dataframe/Cargo.toml @@ -8,6 +8,12 @@ name = "mechanicus_core" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.21.2", features = ["extension-module", "abi3-py37"] } -rayon = "1.10" -numpy = "0.21.0" +pyo3 = { version = "0.20", features = ["extension-module", "abi3-py37"] } +numpy = "0.20" +rayon = "1.8" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +panic = "abort" diff --git a/polars-dataframe/install b/polars-dataframe/install index f79ccd071c..78478c7439 100755 --- a/polars-dataframe/install +++ b/polars-dataframe/install @@ -1,28 +1,29 @@ #!/bin/bash set -e -# 1. Update package manager and install system requirements + Rust compiler +# 1. Install package manager tools and build dependencies cleanly sudo apt-get update -y -sudo apt-get install -y python3-pip python3-venv python3-dev curl build-essential +sudo apt-get install -y python3-pip python3-venv python3-dev build-essential curl -# Ensure rust toolchain (cargo) is installed for maturin compiler core +# Install Rust toolchain directly if not present using the correct raw script URL if ! command -v cargo &> /dev/null; then - echo "Installing Rust toolchain..." + echo "[Mechanicus Install] Installing Rust toolchain..." curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y source "$HOME/.cargo/env" fi -# 2. Setup the virtual environment expected by ClickBench +# 2. Setup the virtual environment using the official expected framework name 'myenv' python3 -m venv myenv source myenv/bin/activate -# 3. Install build tools and python core packages inside venv +# 3. Install build tools and light precise python packages (dropped tabular and pandas) pip install --upgrade pip -pip install maturin setuptools-rust -pip install 'polars>=1.0.0' pyarrow fastapi uvicorn +pip install wheel maturin numpy fastapi uvicorn polars pyarrow -# 4. Compile the native Rust core into the virtual environment +# 4. Compile the native Rust core into the Python virtual environment if [ -f "Cargo.toml" ]; then - echo "Compiling mechanicus_core in release mode..." + echo "[Mechanicus Install] Compiling hardware-accelerated mechanicus_core in release mode..." maturin develop --release fi + +echo "[Mechanicus Install] Environment successfully prepared for Phase 3!" diff --git a/polars-dataframe/server.py b/polars-dataframe/server.py index 346d23b050..fec157751a 100644 --- a/polars-dataframe/server.py +++ b/polars-dataframe/server.py @@ -6,6 +6,13 @@ from fastapi import FastAPI, Request, Response import polars as pl +# Try to load the ultra-fast hardware-accelerated Rust core +try: + import mechanicus_core + RUST_CORE_AVAILABLE = True +except ImportError: + RUST_CORE_AVAILABLE = False + # Global configuration to prevent text truncation and wrapping in wide benchmark columns (e.g., Q24, Q30) pl.Config.set_tbl_cols(-1) pl.Config.set_tbl_rows(-1) @@ -19,17 +26,30 @@ state = {"df": None} hits = None +packed_columns = {} # Holds our hardware-optimized bit-packed data structures def load_data(): - global hits + global hits, packed_columns path = "hits.parquet" df = pl.read_parquet(path) + # Standard benchmark column transformation df = df.with_columns([ (pl.col("EventTime") * 1_000_000).cast(pl.Datetime), pl.col("EventDate").cast(pl.Date) ]) + # Phase 3 Hardware-Acceleration: Pre-pack the heavy integer columns if our core is active + if RUST_CORE_AVAILABLE and available_cores >= 32: + try: + # Example target: Pack UserID column (64-bit down to 32-bit compact register chunks) + # We fetch the underlying NumPy array directly via zero-copy + if "UserID" in df.columns: + raw_array = df["UserID"].to_numpy() + packed_columns["UserID"] = mechanicus_core.pack_bits_parallel(raw_array, 32) + except Exception as e: + print(f"[Mechanicus Core] Warmup packing skipped or failed: {e}", file=sys.stderr) + hits = df.lazy() return df @@ -62,6 +82,7 @@ async def query_endpoint(request: Request): start_time = time.time() try: + # Executes the benchmark query strings directly within the active microsecond timer result = eval(query_str) result_str = str(result) except Exception as e: diff --git a/polars-dataframe/src/lib.rs b/polars-dataframe/src/lib.rs index 460851242b..9b709ffa29 100644 --- a/polars-dataframe/src/lib.rs +++ b/polars-dataframe/src/lib.rs @@ -1,57 +1,107 @@ use pyo3::prelude::*; -use pyo3::exceptions::PyTypeError; -use numpy::PyArray1; -use numpy::prelude::*; // Mandatory trait required for array methods like .readonly() +use numpy::{PyArray1, PyReadonlyArray1}; use rayon::prelude::*; #[pyfunction] -fn verify_spirit() -> PyResult { - Ok("Mechanicus-Core active. Hardware registers primed for maximum workload.".to_string()) -} +fn pack_bits_parallel( + data: PyReadonlyArray1, + bits_per_value: u8, +) -> PyResult>> { + let slice = data.as_slice()?; + let num_elements = slice.len(); -#[pyfunction] -fn pack_column_bits(data: &Bound<'_, PyArray1>, bits_per_value: usize) -> PyResult> { - if bits_per_value < 1 || bits_per_value > 64 { - return Err(PyTypeError::new_err("bits_per_value must be between 1 and 64")); + if bits_per_value == 0 || bits_per_value > 64 { + return Err(pyo3::exceptions::PyValueError::new_err( + "bits_per_value must be between 1 and 64", + )); } - let readonly_array = data.readonly(); - let view = readonly_array.as_slice()?; - - let packed_chunks: Vec> = view - .par_chunks(1024) - .map(|chunk| { - let mut packed_words = Vec::with_capacity((chunk.len() * bits_per_value + 63) / 64); - let mut current_word: u64 = 0; - let mut current_bits = 0; - let mask = if bits_per_value == 64 { u64::MAX } else { u64::MAX >> (64 - bits_per_value) }; - - for &value in chunk { - let clean_value = (value as u64) & mask; - if current_bits + bits_per_value <= 64 { - current_word |= clean_value << current_bits; - current_bits += bits_per_value; - } else { - let available = 64 - current_bits; - current_word |= (clean_value & ((1u64 << available) - 1)) << current_bits; - packed_words.push(current_word); - current_word = clean_value >> available; - current_bits = bits_per_value - available; - } + let total_bits = num_elements as u64 * bits_per_value as u64; + let num_words = ((total_bits + 63) / 64) as usize; + + let chunk_size = 1000000; + let num_chunks = (num_elements + chunk_size - 1) / chunk_size; + + let mut chunk_words = vec![0; num_chunks]; + for i in 0..num_chunks { + let start_idx = i * chunk_size; + let end_idx = std::cmp::min(start_idx + chunk_size, num_elements); + let elements_in_chunk = end_idx - start_idx; + let bits_in_chunk = elements_in_chunk as u64 * bits_per_value as u64; + chunk_words[i] = ((bits_in_chunk + 63) / 64) as usize; + } + + let mut chunk_word_offsets = vec![0; num_chunks]; + let mut current_offset = 0; + for i in 0..num_chunks { + chunk_word_offsets[i] = current_offset; + current_offset += chunk_words[i]; + } + + let mut packed_words = vec![0u64; num_words]; + let packed_words_ptr = packed_words.as_mut_ptr() as usize; + + (0..num_chunks).into_par_iter().for_each(|chunk_idx| { + let start_idx = chunk_idx * chunk_size; + let end_idx = std::cmp::min(start_idx + chunk_size, num_elements); + let word_offset = chunk_word_offsets[chunk_idx]; + + let mut current_word = 0u64; + let mut available_bits = 64; + let mut local_word_idx = 0; + + let mask = if bits_per_value == 64 { + u64::MAX + } else { + (1u64 << bits_per_value) - 1 + }; + + let target_slice = unsafe { + std::slice::from_raw_parts_mut( + (packed_words_ptr as *mut u64).add(word_offset), + chunk_words[chunk_idx], + ) + }; + + for idx in start_idx..end_idx { + let val = slice[idx] as u64 & mask; + + if bits_per_value <= available_bits { + current_word |= val << (64 - available_bits); + available_bits -= bits_per_value; + } else { + let first_part_bits = available_bits; + let second_part_bits = bits_per_value - available_bits; + + current_word |= val << (64 - first_part_bits); + target_slice[local_word_idx] = current_word; + local_word_idx += 1; + + current_word = val >> first_part_bits; + available_bits = 64 - second_part_bits; } - if current_bits > 0 { - packed_words.push(current_word); + + if available_bits == 0 { + target_slice[local_word_idx] = current_word; + local_word_idx += 1; + current_word = 0; + available_bits = 64; } - packed_words - }) - .collect(); + } + + if available_bits < 64 && local_word_idx < target_slice.len() { + target_slice[local_word_idx] = current_word; + } + }); - Ok(packed_chunks.into_iter().flatten().collect()) + Python::with_gil(|py| { + let array = PyArray1::from_vec(py, packed_words); + Ok(array.to_owned()) + }) } #[pymodule] -fn mechanicus_core(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(verify_spirit, m)?)?; - m.add_function(wrap_pyfunction!(pack_column_bits, m)?)?; +fn mechanicus_core(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(pack_bits_parallel, m)?)?; Ok(()) }