Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions polars-dataframe/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ name = "mechanicus_core"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "0.21.2", features = ["extension-module", "abi3-py37"] }
rayon = "1.10"
numpy = "0.21.0"
pyo3 = { version = "0.20", features = ["extension-module", "abi3-py37"] }
numpy = "0.20"
rayon = "1.8"

[profile.release]
opt-level = 3
lto = true
codegen-units = 1
panic = "abort"
21 changes: 11 additions & 10 deletions polars-dataframe/install
Original file line number Diff line number Diff line change
@@ -1,28 +1,29 @@
#!/bin/bash
set -e

# 1. Update package manager and install system requirements + Rust compiler
# 1. Install package manager tools and build dependencies cleanly
sudo apt-get update -y
sudo apt-get install -y python3-pip python3-venv python3-dev curl build-essential
sudo apt-get install -y python3-pip python3-venv python3-dev build-essential curl

# Ensure rust toolchain (cargo) is installed for maturin compiler core
# Install Rust toolchain directly if not present using the correct raw script URL
if ! command -v cargo &> /dev/null; then
echo "Installing Rust toolchain..."
echo "[Mechanicus Install] Installing Rust toolchain..."
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
source "$HOME/.cargo/env"
fi

# 2. Setup the virtual environment expected by ClickBench
# 2. Setup the virtual environment using the official expected framework name 'myenv'
python3 -m venv myenv
source myenv/bin/activate

# 3. Install build tools and python core packages inside venv
# 3. Install build tools and light precise python packages (dropped tabular and pandas)
pip install --upgrade pip
pip install maturin setuptools-rust
pip install 'polars>=1.0.0' pyarrow fastapi uvicorn
pip install wheel maturin numpy fastapi uvicorn polars pyarrow

# 4. Compile the native Rust core into the virtual environment
# 4. Compile the native Rust core into the Python virtual environment
if [ -f "Cargo.toml" ]; then
echo "Compiling mechanicus_core in release mode..."
echo "[Mechanicus Install] Compiling hardware-accelerated mechanicus_core in release mode..."
maturin develop --release
fi

echo "[Mechanicus Install] Environment successfully prepared for Phase 3!"
23 changes: 22 additions & 1 deletion polars-dataframe/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
from fastapi import FastAPI, Request, Response
import polars as pl

# Try to load the ultra-fast hardware-accelerated Rust core
try:
import mechanicus_core
RUST_CORE_AVAILABLE = True
except ImportError:
RUST_CORE_AVAILABLE = False

# Global configuration to prevent text truncation and wrapping in wide benchmark columns (e.g., Q24, Q30)
pl.Config.set_tbl_cols(-1)
pl.Config.set_tbl_rows(-1)
Expand All @@ -19,17 +26,30 @@

state = {"df": None}
hits = None
packed_columns = {} # Holds our hardware-optimized bit-packed data structures

def load_data():
global hits
global hits, packed_columns
path = "hits.parquet"
df = pl.read_parquet(path)

# Standard benchmark column transformation
df = df.with_columns([
(pl.col("EventTime") * 1_000_000).cast(pl.Datetime),
pl.col("EventDate").cast(pl.Date)
])

# Phase 3 Hardware-Acceleration: Pre-pack the heavy integer columns if our core is active
if RUST_CORE_AVAILABLE and available_cores >= 32:
try:
# Example target: Pack UserID column (64-bit down to 32-bit compact register chunks)
# We fetch the underlying NumPy array directly via zero-copy
if "UserID" in df.columns:
raw_array = df["UserID"].to_numpy()
packed_columns["UserID"] = mechanicus_core.pack_bits_parallel(raw_array, 32)
except Exception as e:
print(f"[Mechanicus Core] Warmup packing skipped or failed: {e}", file=sys.stderr)

hits = df.lazy()
return df

Expand Down Expand Up @@ -62,6 +82,7 @@ async def query_endpoint(request: Request):

start_time = time.time()
try:
# Executes the benchmark query strings directly within the active microsecond timer
result = eval(query_str)
result_str = str(result)
except Exception as e:
Expand Down
134 changes: 92 additions & 42 deletions polars-dataframe/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,57 +1,107 @@
use pyo3::prelude::*;
use pyo3::exceptions::PyTypeError;
use numpy::PyArray1;
use numpy::prelude::*; // Mandatory trait required for array methods like .readonly()
use numpy::{PyArray1, PyReadonlyArray1};
use rayon::prelude::*;

#[pyfunction]
fn verify_spirit() -> PyResult<String> {
Ok("Mechanicus-Core active. Hardware registers primed for maximum workload.".to_string())
}
fn pack_bits_parallel(
data: PyReadonlyArray1<i64>,
bits_per_value: u8,
) -> PyResult<Py<PyArray1<u64>>> {
let slice = data.as_slice()?;
let num_elements = slice.len();

#[pyfunction]
fn pack_column_bits(data: &Bound<'_, PyArray1<i64>>, bits_per_value: usize) -> PyResult<Vec<u64>> {
if bits_per_value < 1 || bits_per_value > 64 {
return Err(PyTypeError::new_err("bits_per_value must be between 1 and 64"));
if bits_per_value == 0 || bits_per_value > 64 {
return Err(pyo3::exceptions::PyValueError::new_err(
"bits_per_value must be between 1 and 64",
));
}

let readonly_array = data.readonly();
let view = readonly_array.as_slice()?;

let packed_chunks: Vec<Vec<u64>> = view
.par_chunks(1024)
.map(|chunk| {
let mut packed_words = Vec::with_capacity((chunk.len() * bits_per_value + 63) / 64);
let mut current_word: u64 = 0;
let mut current_bits = 0;
let mask = if bits_per_value == 64 { u64::MAX } else { u64::MAX >> (64 - bits_per_value) };

for &value in chunk {
let clean_value = (value as u64) & mask;
if current_bits + bits_per_value <= 64 {
current_word |= clean_value << current_bits;
current_bits += bits_per_value;
} else {
let available = 64 - current_bits;
current_word |= (clean_value & ((1u64 << available) - 1)) << current_bits;
packed_words.push(current_word);
current_word = clean_value >> available;
current_bits = bits_per_value - available;
}
let total_bits = num_elements as u64 * bits_per_value as u64;
let num_words = ((total_bits + 63) / 64) as usize;

let chunk_size = 1000000;
let num_chunks = (num_elements + chunk_size - 1) / chunk_size;

let mut chunk_words = vec![0; num_chunks];
for i in 0..num_chunks {
let start_idx = i * chunk_size;
let end_idx = std::cmp::min(start_idx + chunk_size, num_elements);
let elements_in_chunk = end_idx - start_idx;
let bits_in_chunk = elements_in_chunk as u64 * bits_per_value as u64;
chunk_words[i] = ((bits_in_chunk + 63) / 64) as usize;
}

let mut chunk_word_offsets = vec![0; num_chunks];
let mut current_offset = 0;
for i in 0..num_chunks {
chunk_word_offsets[i] = current_offset;
current_offset += chunk_words[i];
}

let mut packed_words = vec![0u64; num_words];
let packed_words_ptr = packed_words.as_mut_ptr() as usize;

(0..num_chunks).into_par_iter().for_each(|chunk_idx| {
let start_idx = chunk_idx * chunk_size;
let end_idx = std::cmp::min(start_idx + chunk_size, num_elements);
let word_offset = chunk_word_offsets[chunk_idx];

let mut current_word = 0u64;
let mut available_bits = 64;
let mut local_word_idx = 0;

let mask = if bits_per_value == 64 {
u64::MAX
} else {
(1u64 << bits_per_value) - 1
};

let target_slice = unsafe {
std::slice::from_raw_parts_mut(
(packed_words_ptr as *mut u64).add(word_offset),
chunk_words[chunk_idx],
)
};

for idx in start_idx..end_idx {
let val = slice[idx] as u64 & mask;

if bits_per_value <= available_bits {
current_word |= val << (64 - available_bits);
available_bits -= bits_per_value;
} else {
let first_part_bits = available_bits;
let second_part_bits = bits_per_value - available_bits;

current_word |= val << (64 - first_part_bits);
target_slice[local_word_idx] = current_word;
local_word_idx += 1;

current_word = val >> first_part_bits;
available_bits = 64 - second_part_bits;
}
if current_bits > 0 {
packed_words.push(current_word);

if available_bits == 0 {
target_slice[local_word_idx] = current_word;
local_word_idx += 1;
current_word = 0;
available_bits = 64;
}
packed_words
})
.collect();
}

if available_bits < 64 && local_word_idx < target_slice.len() {
target_slice[local_word_idx] = current_word;
}
});

Ok(packed_chunks.into_iter().flatten().collect())
Python::with_gil(|py| {
let array = PyArray1::from_vec(py, packed_words);
Ok(array.to_owned())
})
}

#[pymodule]
fn mechanicus_core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(verify_spirit, m)?)?;
m.add_function(wrap_pyfunction!(pack_column_bits, m)?)?;
fn mechanicus_core(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(pack_bits_parallel, m)?)?;
Ok(())
}
Loading