diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 964521ab13..33558db948 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -1,4 +1,6 @@ multimodal +subvariant +Subvariant swmansion executorch RNET @@ -314,3 +316,4 @@ Partitioner denoised ttfa TTFA +agentic \ No newline at end of file diff --git a/apps/nlp/app/index.tsx b/apps/nlp/app/index.tsx index 2e0e024076..4020c1956a 100644 --- a/apps/nlp/app/index.tsx +++ b/apps/nlp/app/index.tsx @@ -20,6 +20,9 @@ export default function Home() { router.navigate('privacy-filter/')}> Privacy Filter + router.navigate('llm/')}> + LLM + ); diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx new file mode 100644 index 0000000000..688886115e --- /dev/null +++ b/apps/nlp/app/llm/index.tsx @@ -0,0 +1,351 @@ +import React, { useRef, useState, type ComponentRef } from 'react'; +import { + View, + Text, + TextInput, + TouchableOpacity, + ScrollView, + StyleSheet, + ActivityIndicator, + KeyboardAvoidingView, + Platform, + Alert, + Image as RNImage, +} from 'react-native'; +import { Skia } from '@shopify/react-native-skia'; +import { + useLLMChatSession, + models, + type ChatMessage, + type GenerationStats, + cv, +} from 'react-native-executorch'; +import ScreenWrapper from '../../components/ScreenWrapper'; +import { getImage, skImageToBuffer } from '../../utils'; + +const MODEL = models.llm.LFM2_5_350M; +const SYSTEM_PROMPT = 'You are a helpful multimodal assistant by Liquid AI.'; +const INITIAL_MESSAGES: ChatMessage[] = [{ role: 'system', content: SYSTEM_PROMPT }]; +const GENERATION_CONFIG = { temperature: 0.7, maxNewTokens: 512, echo: false }; + +type Turn = { + role: 'user' | 'assistant'; + content: string; + imageUri?: string; + stats?: GenerationStats; +}; + +function formatStats(stats: GenerationStats): string { + const decodeMs = stats.inferenceEndMs - stats.firstTokenMs; + const tokensPerSec = (stats.numGeneratedTokens / decodeMs) * 1000; + const totalMs = stats.inferenceEndMs - stats.inferenceStartMs; + const ttftMs = stats.firstTokenMs - stats.inferenceStartMs; + return ( + `${stats.numGeneratedTokens} tokens · ` + + `${tokensPerSec.toFixed(1)} tok/s · ` + + `${ttftMs.toFixed(0)}ms ttft · ` + + `${(totalMs / 1000).toFixed(2)}s` + ); +} + +function LLMContent() { + const { isReady, downloadProgress, error, sendMessage, stop } = useLLMChatSession(MODEL, { + initialMessages: INITIAL_MESSAGES, + generationConfig: GENERATION_CONFIG, + }); + + const [input, setInput] = useState(''); + const [turns, setTurns] = useState([]); + const [streamingResponse, setStreamingResponse] = useState(null); + + const scrollRef = useRef>(null); + const isGenerating = streamingResponse !== null; + + const [attachedImage, setAttachedImage] = useState<{ + uri: string; + name: string; + buffer: cv.ImageBuffer; + } | null>(null); + + const handlePickGalleryImage = async () => { + try { + const uri = await getImage(false); + if (!uri) return; + + const skData = await Skia.Data.fromURI(uri); + if (!skData) throw new Error('Failed to read image file'); + const skImage = Skia.Image.MakeImageFromEncoded(skData); + if (!skImage) throw new Error('Failed to decode image'); + + const buffer = skImageToBuffer(skImage); + setAttachedImage({ uri, buffer, name: `Photo (${skImage.width()}x${skImage.height()})` }); + } catch (err: any) { + Alert.alert('Error', err?.message || 'Failed to select gallery image'); + } + }; + + const handleSend = async () => { + const textMessage = input.trim(); + if ((!textMessage && !attachedImage) || !sendMessage || isGenerating) return; + + const currentImage = attachedImage; + setAttachedImage(null); + setInput(''); + setStreamingResponse(''); + + setTurns((prev) => [ + ...prev, + { role: 'user', content: textMessage, imageUri: currentImage?.uri }, + ]); + + try { + let payload; + if (currentImage) { + payload = [ + { kind: 'image' as const, image: currentImage.buffer }, + textMessage || "What's in this image?", + ]; + } else { + payload = textMessage; + } + + const { response, stats } = await sendMessage(payload, (token) => { + setStreamingResponse((prev) => (prev !== null ? prev + token : token)); + }); + setTurns((prev) => [...prev, { role: 'assistant', content: response as string, stats }]); + } finally { + setStreamingResponse(null); + } + }; + + if (error) { + return ( + + Failed to load model + {error.message} + + ); + } + + if (!isReady) { + return ( + + + + {downloadProgress < 100 + ? `Downloading model… ${downloadProgress.toFixed(0)}%` + : 'Loading model into memory…'} + + + ); + } + + return ( + + scrollRef.current?.scrollToEnd({ animated: false })} + > + {turns.length === 0 && streamingResponse === null && ( + Ask LFM 2.5 VL anything or attach an image. + )} + {turns.map((turn, idx) => ( + + + {turn.imageUri && ( + + )} + + {turn.content || '…'} + + + {turn.stats && {formatStats(turn.stats)}} + + ))} + {streamingResponse !== null && ( + + + {streamingResponse || '…'} + + + )} + + + {attachedImage && ( + + + + + {attachedImage.name} + + + setAttachedImage(null)} + > + ✕ + + + )} + + + + 📷 + + + {isGenerating ? ( + stop?.()}> + Stop + + ) : ( + + Send + + )} + + + ); +} + +export default function LLMScreen() { + return ( + + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: '#f8f9fa' }, + centered: { flex: 1, justifyContent: 'center', alignItems: 'center', padding: 24 }, + loadingText: { marginTop: 16, fontSize: 15, color: '#495057', fontWeight: '600' }, + errorTitle: { fontSize: 16, fontWeight: '700', color: '#e03131', marginBottom: 8 }, + errorBody: { fontSize: 13, color: '#868e96', textAlign: 'center' }, + messages: { flex: 1 }, + messagesContent: { padding: 16, paddingBottom: 8 }, + placeholder: { textAlign: 'center', color: '#adb5bd', marginTop: 40, fontSize: 14 }, + turn: { marginBottom: 12 }, + bubble: { + maxWidth: '85%', + borderRadius: 16, + paddingHorizontal: 14, + paddingVertical: 10, + }, + userBubble: { alignSelf: 'flex-end', backgroundColor: '#0070f3' }, + assistantBubble: { + alignSelf: 'flex-start', + backgroundColor: '#fff', + borderWidth: 1, + borderColor: '#e9ecef', + }, + userText: { color: '#fff', fontSize: 15, lineHeight: 21 }, + assistantText: { color: '#212529', fontSize: 15, lineHeight: 21 }, + turnThumbnail: { + width: 160, + height: 120, + borderRadius: 8, + marginBottom: 8, + backgroundColor: '#e9ecef', + }, + statsLine: { + alignSelf: 'flex-start', + marginTop: 4, + marginLeft: 4, + fontSize: 11, + color: '#adb5bd', + }, + attachmentBar: { + flexDirection: 'row', + alignItems: 'center', + paddingHorizontal: 16, + paddingVertical: 8, + backgroundColor: '#fff', + borderTopWidth: 1, + borderTopColor: '#e9ecef', + gap: 12, + }, + attachmentPreview: { + width: 40, + height: 40, + borderRadius: 6, + backgroundColor: '#e9ecef', + }, + attachmentInfo: { flex: 1 }, + attachmentName: { fontSize: 13, fontWeight: '500', color: '#212529' }, + removeAttachmentButton: { + paddingHorizontal: 8, + paddingVertical: 4, + borderRadius: 12, + backgroundColor: '#f1f3f5', + }, + removeAttachmentText: { fontSize: 13, fontWeight: '700', color: '#868e96' }, + inputRow: { + flexDirection: 'row', + alignItems: 'flex-end', + padding: 12, + gap: 8, + borderTopWidth: 1, + borderTopColor: '#e9ecef', + backgroundColor: '#fff', + }, + galleryButton: { + width: 44, + height: 44, + backgroundColor: '#f1f3f5', + borderRadius: 22, + justifyContent: 'center', + alignItems: 'center', + }, + galleryButtonText: { fontSize: 18 }, + input: { + flex: 1, + backgroundColor: '#f1f3f5', + borderRadius: 20, + paddingHorizontal: 16, + paddingVertical: 10, + fontSize: 15, + color: '#212529', + maxHeight: 120, + }, + sendButton: { + backgroundColor: '#0070f3', + borderRadius: 20, + paddingHorizontal: 18, + height: 44, + justifyContent: 'center', + alignItems: 'center', + }, + sendButtonDisabled: { backgroundColor: '#a3cdff' }, + stopButton: { backgroundColor: '#e03131' }, + sendButtonText: { color: '#fff', fontSize: 15, fontWeight: '600' }, +}); diff --git a/apps/nlp/components/ModelPicker.tsx b/apps/nlp/components/ModelPicker.tsx new file mode 100644 index 0000000000..ae3893eb2c --- /dev/null +++ b/apps/nlp/components/ModelPicker.tsx @@ -0,0 +1,81 @@ +import React from 'react'; +import { View, Text, ScrollView, Pressable, StyleSheet } from 'react-native'; + +export type ModelOption = { + label: string; + value: any; + labels?: any; +}; + +interface ModelPickerProps { + label: string; + options: ModelOption[]; + selectedValue: any; + onValueChange: (value: any) => void; +} + +export function ModelPicker({ label, options, selectedValue, onValueChange }: ModelPickerProps) { + return ( + + {label} + + {options.map((option, index) => { + const isSelected = option.value === selectedValue; + return ( + onValueChange(option.value)} + > + + {option.label} + + + ); + })} + + + ); +} + +const styles = StyleSheet.create({ + container: { + width: '100%', + marginBottom: 16, + paddingHorizontal: 4, + }, + label: { + fontSize: 12, + fontWeight: '700', + color: '#666', + textTransform: 'uppercase', + marginBottom: 8, + letterSpacing: 0.5, + }, + scrollContainer: { + flexDirection: 'row', + gap: 8, + }, + chip: { + backgroundColor: '#e0e0e0', + paddingHorizontal: 14, + paddingVertical: 8, + borderRadius: 20, + }, + activeChip: { + backgroundColor: '#000', + }, + chipText: { + fontSize: 13, + color: '#333', + fontWeight: '500', + }, + activeChipText: { + color: '#fff', + fontWeight: '600', + }, +}); diff --git a/apps/nlp/package.json b/apps/nlp/package.json index d35e45a6af..50394f8c65 100644 --- a/apps/nlp/package.json +++ b/apps/nlp/package.json @@ -3,6 +3,12 @@ "version": "1.0.0", "main": "expo-router/entry", "react-native-executorch": { + "backends": [ + "xnnpack", + "coreml", + "mlx", + "vulkan" + ], "features": [ "tokenizer", "textEmbeddings", @@ -21,9 +27,12 @@ "dependencies": { "@react-navigation/drawer": "^7.9.4", "@react-navigation/native": "^7.2.2", + "@shopify/react-native-skia": "2.6.2", "expo": "~56.0.9", "expo-build-properties": "~56.0.17", "expo-constants": "~56.0.17", + "expo-image-manipulator": "~56.0.18", + "expo-image-picker": "~56.0.18", "expo-linking": "~56.0.13", "expo-router": "~56.2.9", "react": "19.2.3", diff --git a/apps/nlp/utils.ts b/apps/nlp/utils.ts new file mode 100644 index 0000000000..a8271596c4 --- /dev/null +++ b/apps/nlp/utils.ts @@ -0,0 +1,75 @@ +import { Alert } from 'react-native'; +import * as ImagePicker from 'expo-image-picker'; +import { ImageManipulator, SaveFormat } from 'expo-image-manipulator'; +import type { SkImage } from '@shopify/react-native-skia'; +import type { cv } from 'react-native-executorch'; + +/** + * Converts a Skia image into the raw RGBA/HWC image buffer that + * react-native-executorch vision tasks accept. Throws if the pixel data cannot + * be read. + * @param image - The Skia image to read pixels from. + * @returns The RGBA/HWC image buffer with its `data`, `width`, `height`, + * `format`, and `layout`. + */ +export const skImageToBuffer = (image: SkImage): cv.ImageBuffer => { + const pixels = image.readPixels(); + if (!pixels) { + throw new Error('Failed to read pixels from image'); + } + if (!(pixels instanceof Uint8Array)) { + throw new Error('Expected Uint8Array from readPixels'); + } + return { + data: pixels, + width: image.width(), + height: image.height(), + format: 'rgba' as const, + layout: 'hwc' as const, + }; +}; + +export const getImage = async ( + useCamera: boolean, + targetWidth = 800 +): Promise => { + const permissionResult = useCamera + ? await ImagePicker.requestCameraPermissionsAsync() + : await ImagePicker.requestMediaLibraryPermissionsAsync(); + + if (!permissionResult.granted) { + Alert.alert( + 'Permission Required', + useCamera + ? 'Permission to access camera is required!' + : 'Permission to access camera roll is required!' + ); + return; + } + + const options: ImagePicker.ImagePickerOptions = { + mediaTypes: ['images'], + allowsEditing: false, + quality: 1, + }; + + const pickerResult = useCamera + ? await ImagePicker.launchCameraAsync(options) + : await ImagePicker.launchImageLibraryAsync(options); + + if (pickerResult.canceled || !pickerResult.assets[0]) { + return; + } + + const asset = pickerResult.assets[0]; + let imageRef = await ImageManipulator.manipulate(asset.uri).renderAsync(); + + if (imageRef.width > targetWidth) { + imageRef = await ImageManipulator.manipulate(asset.uri) + .resize({ width: targetWidth }) + .renderAsync(); + } + + const result = await imageRef.saveAsync({ format: SaveFormat.PNG }); + return result.uri; +}; diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt index 0212e869f8..3030946c35 100644 --- a/packages/react-native-executorch/android/CMakeLists.txt +++ b/packages/react-native-executorch/android/CMakeLists.txt @@ -27,6 +27,7 @@ file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp) file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp) file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp) file(GLOB SPEECH_SOURCES ${CPP_DIR}/extensions/speech/*.cpp) +file(GLOB LLM_SOURCES ${CPP_DIR}/extensions/llm/*.cpp) file(GLOB OPENCV_SOURCES ${CPP_DIR}/extensions/cv/*.cpp) set(RNE_SOURCES @@ -35,6 +36,7 @@ set(RNE_SOURCES ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES} + ${LLM_SOURCES} cpp-adapter.cpp ) diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp index df1bcb9d4d..98fb8c97ff 100644 --- a/packages/react-native-executorch/cpp/RnExecutorch.cpp +++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp @@ -1,6 +1,7 @@ #include "RnExecutorch.h" #include "core/install.h" +#include "extensions/llm/install.h" #include "extensions/math/install.h" #include "extensions/nlp/install.h" #include "extensions/speech/install.h" @@ -22,6 +23,7 @@ void install(jsi::Runtime &jsiRuntime) { rnexecutorch::extensions::math::install(jsiRuntime, module); rnexecutorch::extensions::nlp::install(jsiRuntime, module); rnexecutorch::extensions::speech::install(jsiRuntime, module); + rnexecutorch::extensions::llm::install(jsiRuntime, module); jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module)); } diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.cpp b/packages/react-native-executorch/cpp/extensions/llm/install.cpp new file mode 100644 index 0000000000..f30a2bd80a --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.cpp @@ -0,0 +1,14 @@ +#include "install.h" +#include "llm_runner.h" + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; + +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) { + jsi::Object llmModule = jsi::Object(rt); + + install_createLLMRunner(rt, llmModule); + + module.setProperty(rt, "llm", llmModule); +} +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.h b/packages/react-native-executorch/cpp/extensions/llm/install.h new file mode 100644 index 0000000000..89d3de5e6b --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +namespace rnexecutorch::extensions::llm { +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp new file mode 100644 index 0000000000..ce32532593 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -0,0 +1,344 @@ +// Upstream ExecuTorch annotates experimental LLM APIs (MultimodalRunner, Stats, +// load_tokenizer, create_multimodal_runner) with `ET_EXPERIMENTAL`, which +// expands to `[[deprecated("...")]]`. We suppress -Wdeprecated-declarations so +// Clang does not fail builds on ExecuTorch's experimental API tags. +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include "llm_runner.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "core/conversions.h" +#include "core/error.h" +#include "core/tensor_helpers.h" + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; +namespace error = rnexecutorch::core::error; +namespace tensor = rnexecutorch::core::tensor; +namespace conversions = rnexecutorch::core::conversions; + +using rnexecutorch::core::types::DType; + +namespace { +jsi::Object statsToJSI(jsi::Runtime &rt, const executorch::extension::llm::Stats &stats) { + jsi::Object obj(rt); + obj.setProperty(rt, "numPromptTokens", static_cast(stats.num_prompt_tokens)); + obj.setProperty(rt, "numGeneratedTokens", static_cast(stats.num_generated_tokens)); + obj.setProperty(rt, "firstTokenMs", static_cast(stats.first_token_ms)); + obj.setProperty(rt, "inferenceStartMs", static_cast(stats.inference_start_ms)); + obj.setProperty(rt, "inferenceEndMs", static_cast(stats.inference_end_ms)); + obj.setProperty(rt, "modelLoadStartMs", static_cast(stats.model_load_start_ms)); + obj.setProperty(rt, "modelLoadEndMs", static_cast(stats.model_load_end_ms)); + return obj; +} + +std::vector parsePrompt( + jsi::Runtime &rt, + const std::string &ctx, + const jsi::Value &value, + const std::vector &supportedModalities) { + + if (value.isString()) { + return {executorch::extension::llm::MultimodalInput(conversions::asType(rt, ctx, value))}; + } + + auto arr = conversions::asType(rt, ctx, value); + size_t len = arr.length(rt); + + std::vector inputs; + inputs.reserve(len); + + for (size_t i = 0; i < len; ++i) { + auto elem = arr.getValueAtIndex(rt, i); + std::string itemCtx = std::format("{}[{}]", ctx, i); + + if (elem.isString()) { + inputs.emplace_back(conversions::asType(rt, itemCtx, elem)); + continue; + } + auto mediaObj = conversions::asType(rt, itemCtx, elem); + auto kind = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "kind"); + + if (std::ranges::find(supportedModalities, kind) == supportedModalities.end()) { + throw error::InvalidArgument(std::format("{}: Modality '{}' is not supported " + "by this runner instance", + itemCtx, kind)); + } + + if (kind == "image") { + auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "image"); + auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"C", "H", "W"}); + auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); + + const auto &shape = tensorHost->shape_; + const auto C = shape[0]; + const auto H = shape[1]; + const auto W = shape[2]; + + std::vector data(tensorHost->numel_); + std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); + inputs.emplace_back(executorch::extension::llm::Image(std::move(data), W, H, C)); + } else if (kind == "audio") { + auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "audio"); + auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"batch", "n_bins", "n_frames"}); + auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); + + const auto &shape = tensorHost->shape_; + const auto batchSize = shape[0]; + const auto nBins = shape[1]; + const auto nFrames = shape[2]; + + std::vector data(tensorHost->numel_); + std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); + inputs.emplace_back(executorch::extension::llm::Audio(std::move(data), batchSize, nBins, nFrames)); + } else { + throw error::InvalidArgument(std::format("{}: Unsupported media kind '{}'", itemCtx, kind)); + } + } + + return inputs; +} +} // namespace + +LLMRunnerHostObject::LLMRunnerHostObject(const std::string &modelPath, + const std::string &tokenizerPath, + const std::vector &modalities) + : modelPath_(modelPath), + tokenizerPath_(tokenizerPath), + modalities_(modalities) { + + auto tokenizer = executorch::extension::llm::load_tokenizer(tokenizerPath); + if (!tokenizer) { + throw error::LoadFailed(std::format("LLMRunner: Failed to load runner tokenizer at path: {}", tokenizerPath)); + } + + if (modalities_.empty()) { + runner_ = executorch::extension::llm::create_text_llm_runner(modelPath, std::move(tokenizer)); + } else { + runner_ = executorch::extension::llm::create_multimodal_runner(modelPath, std::move(tokenizer)); + } + + if (!runner_) { + throw error::LoadFailed("LLMRunner: Failed to create llm runner"); + } + + auto loadError = runner_->load(); + if (loadError != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(loadError); + throw error::LoadFailed(std::format("LLMRunner: Failed to load model: {}", errorMsg), loadError); + } +} + +jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { + auto nameStr = name.utf8(rt); + + if (nameStr == "modelPath") { + return jsi::String::createFromUtf8(rt, modelPath_); + } + + if (nameStr == "tokenizerPath") { + return jsi::String::createFromUtf8(rt, tokenizerPath_); + } + + if (nameStr == "modalities") { + return conversions::toJsiArray(rt, modalities_); + } + + if (nameStr == "generate") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 1) { + throw error::InvalidArgument("LLMRunner.generate: Usage: generate(prompt, config?, onToken?)"); + } + + executorch::extension::llm::GenerationConfig config; + if (count > 1 && !args[1].isUndefined() && !args[1].isNull()) { + auto configObj = conversions::asType(rt, "LLMRunner.generate: config", args[1]); + if (auto echoOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "echo")) { + config.echo = *echoOpt; + } + if (auto ignoreEosOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "ignoreEos")) { + config.ignore_eos = *ignoreEosOpt; + } + if (auto maxNewTokensOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "maxNewTokens")) { + config.max_new_tokens = *maxNewTokensOpt; + } + if (auto tempOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "temperature")) { + config.temperature = *tempOpt; + } + } + + std::function tokenCallback; + if (count > 2 && !args[2].isUndefined() && !args[2].isNull()) { + auto tokenFn = std::make_shared(conversions::asType(rt, "LLMRunner.generate: onToken", args[2])); + tokenCallback = [&rt, tokenFn](const std::string &token) { + tokenFn->call(rt, jsi::String::createFromUtf8(rt, token)); + }; + } + + auto finalStats = std::make_shared(); + auto statsCallback = [finalStats](const executorch::extension::llm::Stats &stats) { + finalStats->num_prompt_tokens = stats.num_prompt_tokens; + finalStats->num_generated_tokens = stats.num_generated_tokens; + finalStats->first_token_ms = stats.first_token_ms; + finalStats->inference_start_ms = stats.inference_start_ms; + finalStats->inference_end_ms = stats.inference_end_ms; + finalStats->model_load_start_ms = stats.model_load_start_ms; + finalStats->model_load_end_ms = stats.model_load_end_ms; + finalStats->aggregate_sampling_time_ms = stats.aggregate_sampling_time_ms; + }; + + // Hold the lock for the whole call so dispose() cannot free the + // runner mid-generation (dispose blocks on this lock until we + // return). try_to_lock: only one prefill/generate may run at a + // time, so fail fast instead of queuing. stop() is lock-free and + // can still interrupt us. + std::unique_lock lock(self->mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + throw error::ResourceBusy("LLMRunner.generate: Runner is already in use"); + } + if (!self->runner_) { + throw error::ResourceDisposed("LLMRunner.generate: Runner has been disposed"); + } + + auto genError = executorch::runtime::Error::Ok; + if (self->modalities_.empty()) { + auto prompt = conversions::asType(rt, "LLMRunner.generate: prompt", args[0]); + genError = self->runner_->generate(prompt, config, tokenCallback, statsCallback); + } else { + auto inputs = parsePrompt(rt, "LLMRunner.generate", args[0], self->modalities_); + auto *multimodalRunner = dynamic_cast(self->runner_.get()); + if (multimodalRunner == nullptr) { + throw error::InvalidArgument("LLMRunner.generate: Runner instance is not a multimodal model"); + } + genError = multimodalRunner->generate(inputs, config, tokenCallback, statsCallback); + } + + if (genError != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(genError); + throw error::ExecutionFailed(std::format("LLMRunner.generate: Failed to generate: {}", errorMsg), genError); + } + + return statsToJSI(rt, *finalStats); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "generate"), 1, error::guarded(fnBody)); + } + + if (nameStr == "prefill") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 1) { + throw error::InvalidArgument("LLMRunner.prefill: Usage: prefill(prompt)"); + } + + // Lock held for the whole call, same as generate(). + std::unique_lock lock(self->mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + throw error::ResourceBusy("LLMRunner.prefill: Runner is already in use"); + } + if (!self->runner_) { + throw error::ResourceDisposed("LLMRunner.prefill: Runner has been disposed"); + } + + auto inputs = parsePrompt(rt, "LLMRunner.prefill", args[0], self->modalities_); + auto result = self->runner_->prefill(inputs); + + if (!result.ok()) { + std::string errorMsg = executorch::runtime::to_string(result.error()); + throw error::ExecutionFailed(std::format("LLMRunner.prefill: Failed: {}", errorMsg), result.error()); + } + + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "prefill"), 1, error::guarded(fnBody)); + } + + if (nameStr == "stop") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime & /*rt*/, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t /*count*/) -> jsi::Value { + // Intentionally no mutex here: stop() is designed to be called + // concurrently to interrupt an in-progress generate(). Taking the + // lock would block until generate() finishes, defeating the point. + // runner_ is only cleared by dispose() on this same (JS) thread, + // so reading it lock-free here is safe. + if (!self->runner_) { + throw error::ResourceDisposed("LLMRunner.stop: Runner has been disposed"); + } + self->runner_->stop(); + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "stop"), 0, error::guarded(fnBody)); + } + + if (nameStr == "dispose") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime & /*rt*/, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { + if (count != 0) { + throw error::InvalidArgument("dispose: Usage: dispose()"); + } + + // Signal stop before locking so any in-progress generate() exits + // quickly; we then block on the lock until it returns and clear + // the runner, which frees the model. Idempotent: a second + // dispose() finds a null runner_ and is a no-op. + if (self->runner_) { + self->runner_->stop(); + } + + std::unique_lock lock(self->mutex_); + self->runner_ = nullptr; + + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, error::guarded(fnBody)); + } + + return jsi::Value::undefined(); +} + +std::vector LLMRunnerHostObject::getPropertyNames(jsi::Runtime &rt) { + std::vector properties; + properties.push_back(jsi::PropNameID::forAscii(rt, "modelPath")); + properties.push_back(jsi::PropNameID::forAscii(rt, "tokenizerPath")); + properties.push_back(jsi::PropNameID::forAscii(rt, "modalities")); + properties.push_back(jsi::PropNameID::forAscii(rt, "prefill")); + properties.push_back(jsi::PropNameID::forAscii(rt, "generate")); + properties.push_back(jsi::PropNameID::forAscii(rt, "stop")); + properties.push_back(jsi::PropNameID::forAscii(rt, "dispose")); + return properties; +} + +void install_createLLMRunner(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "createLLMRunner"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 2 || count > 3) { + throw error::InvalidArgument("createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath, modalities?)"); + } + + auto modelPath = conversions::asType(rt, "createLLMRunner: modelPath", args[0]); + auto tokenizerPath = conversions::asType(rt, "createLLMRunner: tokenizerPath", args[1]); + + std::vector modalities; + if (count > 2 && !args[2].isNull() && !args[2].isUndefined()) { + modalities = conversions::asVector(rt, "createLLMRunner: modalities", args[2]); + } + + auto runnerInstance = std::make_shared(modelPath, tokenizerPath, std::move(modalities)); + return jsi::Object::createFromHostObject(rt, runnerInstance); + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, error::guarded(fnBody))); +} +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h new file mode 100644 index 0000000000..74e0dbf5b0 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -0,0 +1,66 @@ +#pragma once + +// Upstream ExecuTorch annotates experimental LLM APIs (TextLLMRunner, Stats, +// etc.) with `ET_EXPERIMENTAL`, which expands to `[[deprecated("This API is +// experimental...")]]`. We suppress -Wdeprecated-declarations so Clang does not +// fail builds on ExecuTorch's experimental API tags. +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include +#include +#include +#include + +#include + +#include + +namespace rnexecutorch::extensions::llm { +/** + * JSI HostObject wrapping an ExecuTorch LLM runner instance + * (`executorch::extension::llm::IRunner`). + * + * Exposes methods to JavaScript for prefilling prompt context, generating token + * stream continuations, interrupting generation, and releasing native model + * memory. + */ +class LLMRunnerHostObject : public facebook::jsi::HostObject, + public std::enable_shared_from_this { +public: + /** + * Constructs an LLMRunnerHostObject by loading an ExecuTorch model binary + * and tokenizer. + * + * @param modelPath Absolute file system path to the `.pte` LLM model + * binary. + * @param tokenizerPath Absolute file system path to the local tokenizer + * configuration file (e.g. `tokenizer.json`). + * @param modalities Vector of supported input modality names (e.g. + * `{"image"}`). + * @throws core::error::RnExecuTorchException with code LoadFailed if + * loading the model or tokenizer fails. + */ + LLMRunnerHostObject(const std::string &modelPath, + const std::string &tokenizerPath, + const std::vector &modalities); + + facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; + std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; + +private: + /** Owning pointer to the underlying ExecuTorch IRunner instance. */ + std::unique_ptr runner_; + /** Mutex guarding concurrent access to prefill and generation operations. */ + std::mutex mutex_; + /** File system path to the loaded `.pte` model binary. */ + std::string modelPath_; + /** File system path to the loaded tokenizer configuration file. */ + std::string tokenizerPath_; + /** List of supported non-text input modalities. */ + std::vector modalities_; +}; + +void install_createLLMRunner(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index 2eb9dc91cb..c15604ccea 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -83,6 +83,9 @@ "publishConfig": { "registry": "https://registry.npmjs.org/" }, + "dependencies": { + "@huggingface/jinja": "^0.5.9" + }, "peerDependencies": { "react": "*", "react-native": "*", diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts new file mode 100644 index 0000000000..e6437cad32 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -0,0 +1,201 @@ +import { Template } from '@huggingface/jinja'; + +import { tensor, type Tensor } from '../../core/tensor'; +import { RnExecuTorchError } from '../../core/error'; +import type { ImageBuffer } from '../cv'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../cv/tasks/preprocessing'; +import type { Modality, Prompt, MediaInput } from './llmRunner'; + +/** + * High-level media payload input for chat turns. + * @category Types + */ +export type ChatMediaInput = + | { readonly kind: 'image'; readonly image: ImageBuffer } + | { readonly kind: 'audio'; readonly audio: unknown }; + +/** + * Interleaved text and media content for a chat turn. + * @category Types + */ +export type ChatMessageContent = string | readonly (string | ChatMediaInput)[]; + +/** + * Conversation turn representing system, user, or assistant messages. + * @category Types + */ +export type ChatMessage = { + /** Conversation role ('system', 'user', or 'assistant'). */ + readonly role: 'system' | 'user' | 'assistant'; + /** Message content string or interleaved text and media payloads. */ + readonly content: ChatMessageContent; +}; + +/** + * Image preprocessing and sentinel token config for vision-language LLMs. + * @category Types + */ +export type LLMImagePreprocessorConfig = { + /** Sentinel token delimiters inserted into Jinja prompts. */ + readonly token: { readonly start: string; readonly end: string }; + /** Image preprocessing options (normalization, resize mode, interpolation). */ + readonly preprocessorOpts: ImagePreprocessorOptions; + /** Fixed target shape expected by native LLM `[C, H, W]`. */ + readonly targetShape: readonly [number, number, number]; +}; + +/** + * Audio preprocessing and sentinel token config for audio-language LLMs. + * @category Types + */ +export type LLMAudioPreprocessorConfig = { + /** Sentinel token delimiters inserted into Jinja prompts. */ + readonly token: { readonly start: string; readonly end: string }; +}; + +/** + * Preprocessor configuration for media modalities. + * @category Types + */ +export type LLMMediaPreprocessorConfig = { + readonly image?: LLMImagePreprocessorConfig; + readonly audio?: LLMAudioPreprocessorConfig; +}; + +/** + * Options for instantiating a ChatPreprocessor. + * @category Types + */ +export type ChatPreprocessorConfig = { + readonly chatTemplate: string; + readonly bosToken?: string; + readonly modalities?: readonly Modality[]; + readonly preprocessorConfig?: LLMMediaPreprocessorConfig; +}; + +/** + * Turn preprocessing options for ChatPreprocessor. + * @category Types + */ +export type ChatProcessOptions = { + /** + * Whether this is the initial turn in the conversation (prepends BOS token if + * true). Defaults to `false`. + */ + readonly isFirstTurn?: boolean; + /** + * Whether to append the assistant generation prompt (e.g. + * `<|im_start|>assistant\n`). Defaults to `true`. + */ + readonly addGenerationPrompt?: boolean; +}; + +/** + * Handles Jinja template formatting and media tensor preprocessing for chat turns. + * @category Typescript API + * @param config Preprocessor configuration including template and preprocessor settings. + * @returns Object with process and dispose methods. + */ +export function createChatPreprocessor(config: ChatPreprocessorConfig) { + const { chatTemplate, modalities, preprocessorConfig, bosToken = '' } = config; + const template = new Template(chatTemplate); + + let imgPreprocessor: ReturnType | undefined; + let imgShape: [number, number, number] | undefined; + + if (preprocessorConfig?.image !== undefined) { + imgShape = [...preprocessorConfig.image.targetShape]; + imgPreprocessor = createImagePreprocessor(preprocessorConfig.image.preprocessorOpts, imgShape); + } + + const tensors: Tensor[] = []; + + const dispose = () => { + tensors.forEach((t) => t.dispose()); + imgPreprocessor?.dispose(); + }; + + const process = (message: ChatMessage, opts?: ChatProcessOptions): Prompt => { + const isFirstTurn = opts?.isFirstTurn ?? false; + const addGenerationPrompt = opts?.addGenerationPrompt ?? true; + + if (typeof message.content === 'string') { + /* eslint-disable camelcase */ + return template.render({ + bos_token: isFirstTurn ? bosToken : '', + add_generation_prompt: addGenerationPrompt, + messages: [{ role: message.role, content: message.content }], + }); + /* eslint-enable */ + } + + const mediaInputs: MediaInput[] = []; + let syntheticContent = ''; + + for (const item of message.content) { + if (typeof item === 'string') { + syntheticContent += item; + continue; + } + + if (!modalities || !modalities.includes(item.kind)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Modality '${item.kind}' is not supported by this model instance.` + ); + } + + if (item.kind === 'image') { + if (!preprocessorConfig?.image || !imgPreprocessor || !imgShape) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + 'Received image input but no image preprocessorConfig was provided.' + ); + } + const tokenStart = preprocessorConfig.image.token.start; + const tokenEnd = preprocessorConfig.image.token.end; + const index = mediaInputs.length; + + syntheticContent += `${tokenStart}\uFFFC__ET_MEDIA_${index}__${tokenEnd}`; + + const tImage = tensor('float32', imgShape); + tensors.push(tImage); + imgPreprocessor.process(item.image).copyTo(tImage); + + mediaInputs.push({ kind: 'image', image: tImage }); + } + + if (item.kind === 'audio') { + throw RnExecuTorchError('INVALID_ARGUMENT', 'Audio input not yet supported'); + } + } + + /* eslint-disable camelcase */ + const renderedContent = template.render({ + bos_token: isFirstTurn ? bosToken : '', + add_generation_prompt: addGenerationPrompt, + messages: [{ role: message.role, content: syntheticContent }], + }); + /* eslint-enable */ + + const regex = /\uFFFC__ET_MEDIA_(\d+)__/g; + const prompt: (string | MediaInput)[] = []; + + let match: RegExpExecArray | null; + let lastIndex = 0; + while ((match = regex.exec(renderedContent)) !== null) { + const textChunk = renderedContent.slice(lastIndex, match.index); + if (textChunk.length > 0) prompt.push(textChunk); + + prompt.push(mediaInputs[parseInt(match[1]!, 10)]!); + lastIndex = regex.lastIndex; + } + + const tail = renderedContent.slice(lastIndex); + if (tail.length > 0) prompt.push(tail); + + return prompt; + }; + + return { dispose, process }; +} diff --git a/packages/react-native-executorch/src/extensions/llm/index.ts b/packages/react-native-executorch/src/extensions/llm/index.ts new file mode 100644 index 0000000000..8b338b7712 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/index.ts @@ -0,0 +1,3 @@ +export * from './llmRunner'; +export * from './chatPreprocessor'; +export * from './tokenizerConfig'; diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts new file mode 100644 index 0000000000..dd5246e242 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -0,0 +1,126 @@ +import type { Tensor } from '../../core/tensor'; +import { rnexecutorchJsi } from '../../native/bridge'; + +declare const llmRunnerBrand: unique symbol; + +/** + * Configuration options for LLM text generation. + * @category Types + */ +export type GenerationConfig = { + /** Whether to echo the prompt in the generated output. */ + readonly echo?: boolean; + /** Whether to ignore EOS tokens during generation. */ + readonly ignoreEos?: boolean; + /** Maximum number of new tokens to generate. */ + readonly maxNewTokens?: number; + /** Sampling temperature for token selection. */ + readonly temperature?: number; +}; + +/** + * Execution and performance statistics for a generation call. + * @category Types + */ +export type GenerationStats = { + /** Number of tokens in the input prompt. */ + readonly numPromptTokens: number; + /** Number of newly generated tokens. */ + readonly numGeneratedTokens: number; + /** Time elapsed in milliseconds to generate the first token. */ + readonly firstTokenMs: number; + /** Timestamp in milliseconds when inference started. */ + readonly inferenceStartMs: number; + /** Timestamp in milliseconds when inference completed. */ + readonly inferenceEndMs: number; + /** Timestamp in milliseconds when model loading started. */ + readonly modelLoadStartMs: number; + /** Timestamp in milliseconds when model loading completed. */ + readonly modelLoadEndMs: number; +}; + +/** + * Low-level non-text media input tensor payloads. + * @category Types + */ +export type MediaInput = + | { readonly kind: 'image'; readonly image: Tensor } + | { readonly kind: 'audio'; readonly audio: Tensor }; + +/** + * Supported non-text input modality keys (e.g. `'image'`, `'audio'`). + * @category Types + */ +export type Modality = MediaInput['kind']; + +/** + * Text or interleaved multimodal prompt input for a low-level LLM runner. + * @category Types + */ +export type Prompt = string | readonly (string | MediaInput)[]; + +/** + * Handle to a native ExecuTorch LLM runner. + * @category Types + */ +export type LLMRunner = { + /** Path to the local model file. */ + readonly modelPath: string; + /** Path to the local tokenizer configuration file. */ + readonly tokenizerPath: string; + /** List of supported non-text input modalities for this runner (e.g. `['image']`). */ + readonly modalities: readonly Modality[]; + + /** + * Disposes the native LLM runner and releases the loaded model memory. + */ + dispose(): void; + + /** + * Interrupts and stops any active generation call on this runner. + */ + stop(): void; + + /** + * Prefills the runner with a prompt to build up the KV cache. + * @param prompt The prefill text or multimodal prompt. + */ + prefill(prompt: Prompt): void; + + /** + * Generates text continuation from a prompt. + * @param prompt The text or multimodal prompt to generate continuation for. + * @param config Generation configuration options. + * @param onToken Callback function triggered whenever a new token is generated. + * @returns Generation performance statistics. + */ + generate( + prompt: Prompt, + config?: GenerationConfig, + onToken?: (token: string) => void + ): GenerationStats; + + /** + * Prevents plain JS objects from being cast as LLMRunners. + * @internal + */ + readonly [llmRunnerBrand]: never; +}; + +/** + * Creates a native ExecuTorch LLM runner instance. + * @category Typescript API + * @param modelPath Path to the local `.pte` model file. + * @param tokenizerPath Path to the local tokenizer configuration file (e.g. `tokenizer.json`). + * @param modalities List of supported input non-text modalities (e.g. `['image']`). + * Defaults to text-only. + * @returns A native LLMRunner instance. + */ +export function createLLMRunner( + modelPath: string, + tokenizerPath: string, + modalities?: readonly Modality[] +): LLMRunner { + 'worklet'; + return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath, modalities ?? []); +} diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts new file mode 100644 index 0000000000..1283cf8346 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -0,0 +1,225 @@ +import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; +import RNBlobUtil from 'react-native-blob-util'; + +import { wrapAsync } from '../../../core/runtime'; +import { + createLLMRunner, + type LLMRunner, + type GenerationConfig, + type GenerationStats, + type Modality, + type Prompt, +} from '../llmRunner'; +import { parseTokenizerConfig } from '../tokenizerConfig'; +import { + createChatPreprocessor, + type ChatMediaInput, + type ChatMessageContent, + type ChatMessage, + type LLMImagePreprocessorConfig, + type LLMAudioPreprocessorConfig, + type LLMMediaPreprocessorConfig, +} from '../chatPreprocessor'; + +export type { + GenerationConfig, + GenerationStats, + Modality, + ChatMediaInput, + ChatMessageContent, + ChatMessage, + LLMImagePreprocessorConfig, + LLMAudioPreprocessorConfig, + LLMMediaPreprocessorConfig, +}; + +/** + * Model configuration required to instantiate an LLM chat session. + * @category Types + */ +export type LLMModel = { + /** Local path or remote URL of the `.pte` model file. */ + readonly modelPath: string; + /** Local path or remote URL of the `tokenizer.json` file. */ + readonly tokenizerPath: string; + /** Local path or remote URL of the `tokenizer_config.json` file. */ + readonly tokenizerConfigPath: string; + /** Supported input non-text modalities (e.g. `['image']`). */ + readonly modalities?: readonly Modality[]; + /** Media preprocessor configuration. */ + readonly preprocessorConfig?: LLMMediaPreprocessorConfig; +}; + +/** + * Options for configuring an LLM chat session. + * @category Types + */ +export type LLMChatSessionOptions = { + /** Initial conversation history to prefill into the model KV cache. */ + readonly initialMessages?: readonly ChatMessage[]; + /** Default generation configuration options. */ + readonly generationConfig?: GenerationConfig; + /** Additional stop tokens that interrupt token generation. */ + readonly stopTokens?: readonly string[]; +}; + +/** + * Generation result returned by an LLM chat turn. + * @category Types + */ +export type LLMGenerationResult = { + /** The generated assistant response text. */ + readonly response: ChatMessageContent; + /** Generation performance statistics. */ + readonly stats: GenerationStats; +}; + +/** + * Handle to an active LLM chat session. + * @category Types + */ +export type LLMChatSession = { + /** + * Interrupts and stops any active token generation call. + */ + stop(): void; + + /** + * Releases native model memory and preprocessor resources. + */ + dispose(): void; + + /** + * Returns the read-only conversation message history. + */ + getHistory(): readonly ChatMessage[]; + + /** + * Sends a user message or chat turn to the model and generates a response. + * @param message Message string, media payload array, or ChatMessage object. + * @param onToken Callback fired on the RN thread for each decoded token. + * @param genConfig Generation options overriding session defaults. + * @returns A promise resolving to the response and generation stats. + */ + sendMessage( + message: ChatMessageContent | ChatMessage, + onToken?: (token: string) => void, + genConfig?: GenerationConfig + ): Promise; +}; + +function generateChatTurnWorklet( + runner: LLMRunner, + prompt: Prompt, + options: { + readonly genConfig: GenerationConfig; + readonly stopTokens: readonly string[]; + readonly onToken?: (token: string) => void; + } +): LLMGenerationResult { + 'worklet'; + const { genConfig, stopTokens, onToken } = options; + + let response = ''; + + const callback = (token: string) => { + if (stopTokens.includes(token)) return; + response += token; + if (onToken) scheduleOnRN(onToken, token); + }; + + const stats = runner.generate(prompt, genConfig, callback); + + return { response, stats }; +} + +/** + * Instantiates an LLM chat session using background thread execution. + * @category Typescript API + * @param config Model configuration containing model, tokenizer, and tokenizer config paths. + * @param options Custom generation and state options. + * @param runtime The worklet runtime thread to run native generation on. + * @returns A Promise resolving to an LLMChatSession instance. + */ +export async function createLLMChatSession( + config: LLMModel, + options?: LLMChatSessionOptions, + runtime?: WorkletRuntime +): Promise { + const { modelPath, tokenizerPath, tokenizerConfigPath, modalities, preprocessorConfig } = config; + + const initialMessages = options?.initialMessages ?? []; + const defaultGenerationConfig = options?.generationConfig; + + // Read and parse tokenizer_config.json + const tokenizerConfigStr = await RNBlobUtil.fs.readFile(tokenizerConfigPath, 'utf8'); + const tokenizerConfig = parseTokenizerConfig(JSON.parse(tokenizerConfigStr)); + const { chatTemplate, bosToken, eosToken } = tokenizerConfig; + + // Prepare chat preprocessor + const chatPreprocessor = createChatPreprocessor({ + chatTemplate, + bosToken, + modalities, + preprocessorConfig, + }); + const stopTokens = [...(options?.stopTokens ?? []), ...(eosToken ? [eosToken] : [])]; + + // Prepare runner + const history: ChatMessage[] = []; + const runner = await wrapAsync(createLLMRunner, runtime)(modelPath, tokenizerPath, modalities); + const prefill = wrapAsync(runner.prefill, runtime); + + // Prefill initial messages + for (const msg of initialMessages) { + await prefill( + chatPreprocessor.process(msg, { + isFirstTurn: history.length === 0, + addGenerationPrompt: false, + }) + ); + history.push(msg); + } + + const stop = () => runner.stop(); + const dispose = () => { + runner.dispose(); + chatPreprocessor.dispose(); + }; + + const generateChatTurn = wrapAsync(generateChatTurnWorklet, runtime); + + const sendMessage = async ( + message: ChatMessageContent | ChatMessage, + onToken?: (token: string) => void, + genConfig?: GenerationConfig + ): Promise => { + let input: ChatMessage; + if (typeof message === 'object' && 'role' in message) { + input = message; + } else { + input = { role: 'user', content: message }; + } + + const prompt = chatPreprocessor.process(input, { + isFirstTurn: history.length === 0, + addGenerationPrompt: true, + }); + + history.push(input); + + const opts = { genConfig: { ...defaultGenerationConfig, ...genConfig }, stopTokens, onToken }; + const { response, stats } = await generateChatTurn(runner, prompt, opts); + + history.push({ role: 'assistant', content: response }); + + return { response, stats }; + }; + + return { + stop, + dispose, + sendMessage, + getHistory: () => [...history], + }; +} diff --git a/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts new file mode 100644 index 0000000000..ce07ca4101 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts @@ -0,0 +1,48 @@ +import { RnExecuTorchError } from '../../core/error'; + +/** + * Model chat template configuration resolved from tokenizer config file. + * @category Types + */ +export type TokenizerChatConfig = { + readonly chatTemplate: string; + readonly bosToken?: string; + readonly eosToken?: string; +}; + +function resolveToken(token: unknown): string | undefined { + if (typeof token === 'string') return token; + if (token && typeof token === 'object' && typeof (token as any).content === 'string') { + return (token as any).content; + } + return undefined; +} + +/** + * Parses raw JSON configuration from `tokenizer_config.json` into a normalized format. + * @category Utils + * @param config Raw JSON object from tokenizer_config.json. + * @returns A parsed TokenizerChatConfig object. + */ +export function parseTokenizerConfig(config: any): TokenizerChatConfig { + let chatTemplate = config?.chat_template; + + // Some models ship multiple named templates as `[{ name, template }]`. + if (Array.isArray(chatTemplate)) { + const entry = chatTemplate.find((t) => t?.name === 'default') ?? chatTemplate[0]; + chatTemplate = entry?.template; + } + + if (typeof chatTemplate !== 'string') { + throw RnExecuTorchError( + 'LOAD_FAILED', + 'tokenizer_config.json does not contain a string `chat_template`' + ); + } + + return { + chatTemplate, + bosToken: resolveToken(config?.bos_token), + eosToken: resolveToken(config?.eos_token), + }; +} diff --git a/packages/react-native-executorch/src/fetcher/fetcher.ts b/packages/react-native-executorch/src/fetcher/fetcher.ts index 8bbc9f963a..8af28c4108 100644 --- a/packages/react-native-executorch/src/fetcher/fetcher.ts +++ b/packages/react-native-executorch/src/fetcher/fetcher.ts @@ -1,20 +1,11 @@ /* eslint-disable no-bitwise */ import { Platform } from 'react-native'; import RNBlobUtil from 'react-native-blob-util'; -import * as telemetry from './telemetry'; import { RnExecuTorchError } from '../core/error'; const IS_ANDROID = Platform.OS === 'android'; - -// Persistent, per-app directory where downloaded model assets are cached. -// iOS: internal DocumentDir (not CacheDir) so the OS won't evict large models -// between runs and force a costly re-download. -// Android: the app-private EXTERNAL files dir (getExternalFilesDir), so the -// system DownloadManager can write there and same-volume moves stay cheap -// even for multi-GB files. Falls back to DocumentDir if unmounted. -const ANDROID_DIRECTORY = RNBlobUtil.fs.dirs.SDCardDir || RNBlobUtil.fs.dirs.DocumentDir; -const RNE_DIRECTORY = IS_ANDROID - ? `${ANDROID_DIRECTORY}/react-native-executorch` +const BASE_DIR = IS_ANDROID + ? `${RNBlobUtil.fs.dirs.SDCardDir || RNBlobUtil.fs.dirs.DocumentDir}/react-native-executorch` : `${RNBlobUtil.fs.dirs.DocumentDir}/react-native-executorch`; /** @@ -24,10 +15,7 @@ const RNE_DIRECTORY = IS_ANDROID export interface DownloadOptions { /** Called with overall progress in `[0, 1]` as bytes arrive. */ onProgress?: (progress: number) => void; - /** - * Aborts the download. On iOS the bytes fetched so far are kept on disk so a - * later {@link download} of the same source resumes instead of restarting. - */ + /** Aborts the download. */ signal?: AbortSignal; /** * Re-downloads every remote source even when it is already cached, replacing @@ -37,353 +25,92 @@ export interface DownloadOptions { forceDownload?: boolean; } -// djb2 — cheap, dependency-free hash used to derive a stable cache key per URL. const djb2 = (s: string): number => { let h = 5381; - for (let i = 0; i < s.length; i++) { - h = (((h << 5) + h) ^ s.charCodeAt(i)) >>> 0; - } + for (let i = 0; i < s.length; i++) h = (((h << 5) + h) ^ s.charCodeAt(i)) >>> 0; return h; }; const isRemote = (source: string): boolean => /^https?:\/\//.test(source); function cachePathFor(url: string): string { - const urlWithoutQuery = url.split('?')[0]!; - const basename = urlWithoutQuery.split('/').pop() || 'model'; - // Hash guards against basename collisions across different repos/versions. - return `${RNE_DIRECTORY}/${djb2(urlWithoutQuery)}_${basename}`; -} - -async function fileSize(path: string): Promise { - try { - const stat = await RNBlobUtil.fs.stat(path); - return Number(stat.size) || 0; - } catch { - return 0; - } + const cleanUrl = url.split('?')[0]!; + const filename = cleanUrl.split('/').pop() || 'model'; + return `${BASE_DIR}/${djb2(cleanUrl)}_${filename}`; } -// Best-effort remote content length via a HEAD request; 0 when unknown. -async function remoteSize(url: string): Promise { +async function isDownloaded(destPath: string): Promise { try { - const res = await fetch(url, { method: 'HEAD' }); - const len = res.headers.get('content-length'); - return len ? Number(len) : 0; + const stat = await RNBlobUtil.fs.stat(destPath); + return Number(stat.size) > 0; } catch { - return 0; + return false; } } -// Raised when a download is cancelled through its `signal`. Internal: callers -// match on the DOWNLOAD_ABORTED code via isRnExecuTorchError. -const abortError = () => RnExecuTorchError('DOWNLOAD_ABORTED', 'The download was aborted.'); - -type OnBytes = (received: number, total: number) => void; - -interface DownloadUrlCallbacks { - // Reports absolute received/total bytes for this file, including any bytes - // already present from a previous partial download. - onBytes?: OnBytes; - signal?: AbortSignal; - forceDownload?: boolean; -} - -// A download that is currently running, shared by every caller that asked for -// the same URL while it was in flight. -interface InFlightDownload { - promise: Promise; - // Progress is fanned out to every joined caller. - listeners: Set; - // Drives the underlying request; aborted only once every caller has left. - controller: AbortController; - callers: number; -} - -const inFlight = new Map(); - -// Downloads a single remote file into the cache, dispatching to the -// platform-appropriate backend. Returns the local path. -// -// Concurrent calls for the same URL share one request: without this, both would -// miss the cache check, double-count the download in telemetry, and write the -// same temporary file — on Android the second one's opening `unlink` would -// delete the first one's partially downloaded data. -async function downloadUrl(url: string, cb: DownloadUrlCallbacks): Promise { - if (cb.signal?.aborted) throw abortError(); - - const dest = cachePathFor(url); - - if (cb.forceDownload) { - // Drop the cached copy so the checks below fall through to a real fetch. - // A download already in flight hasn't written `dest` yet, so joining it - // still yields freshly fetched bytes. - await RNBlobUtil.fs.unlink(dest).catch(() => {}); - } else if (await RNBlobUtil.fs.exists(dest)) { - // Cache hit — nothing to download. - const size = await fileSize(dest); - cb.onBytes?.(size, size); - return dest; - } - - // Skip an entry whose last caller just left: it is already unwinding, so - // joining it would hand this caller someone else's cancellation. - const existing = inFlight.get(url); - if (existing && !existing.controller.signal.aborted) return joinDownload(existing, cb); - - const entry: InFlightDownload = { - promise: Promise.resolve(''), - listeners: new Set(), - controller: new AbortController(), - callers: 0, - }; - entry.promise = startDownload(url, dest, entry).finally(() => { - inFlight.delete(url); - }); - inFlight.set(url, entry); - - return joinDownload(entry, cb); -} - -// Runs the actual request, fanning progress out to everyone who joined. -async function startDownload(url: string, dest: string, entry: InFlightDownload): Promise { - // Count this actual (non-cached) fetch once, no matter how many callers share it. - telemetry.triggerHuggingFaceDownloadCounter(url); - telemetry.triggerDownloadEvent(url); - - await RNBlobUtil.fs.mkdir(RNE_DIRECTORY).catch(() => {}); - - const cb: DownloadUrlCallbacks = { - signal: entry.controller.signal, - onBytes: (received, total) => { - for (const listener of entry.listeners) listener(received, total); - }, - }; - - return IS_ANDROID - ? downloadUrlViaAndroidDownloadManager(url, dest, cb) - : downloadUrlViaIosStream(url, dest, cb); -} - -// Attaches one caller to a shared download. The caller's own signal only -// detaches it — the request itself is cancelled once the last caller leaves. -function joinDownload(entry: InFlightDownload, cb: DownloadUrlCallbacks): Promise { - entry.callers += 1; - if (cb.onBytes) entry.listeners.add(cb.onBytes); - - return new Promise((resolve, reject) => { - const onAbort = () => { - if (cb.onBytes) entry.listeners.delete(cb.onBytes); - entry.callers -= 1; - if (entry.callers === 0) entry.controller.abort(); - reject(abortError()); - }; - cb.signal?.addEventListener('abort', onAbort); - - entry.promise.then(resolve, reject).finally(() => { - cb.signal?.removeEventListener('abort', onAbort); - }); - }); -} - -// Android backend: the system DownloadManager streams to app-private external -// storage. Unlike blob-util's in-process reader it handles files larger than -// 2 GB, keeps downloading while the app is in the background or killed, and -// resumes across transient network drops on its own — so no manual Range logic. -async function downloadUrlViaAndroidDownloadManager( +// Download a single file URL directly to disk +async function downloadFile( url: string, - dest: string, - cb: DownloadUrlCallbacks + destPath: string, + onBytes?: (recv: number, tot: number) => void, + signal?: AbortSignal ): Promise { - const tmp = `${dest}.downloading`; - await RNBlobUtil.fs.unlink(tmp).catch(() => {}); + if (signal?.aborted) throw RnExecuTorchError('DOWNLOAD_ABORTED', 'The download was aborted.'); - if (cb.signal?.aborted) throw abortError(); - - const task = RNBlobUtil.config({ - addAndroidDownloads: { - useDownloadManager: true, - path: tmp, - notification: false, - mediaScannable: false, - mime: 'application/octet-stream', - }, - }).fetch('GET', url); + await RNBlobUtil.fs.mkdir(BASE_DIR).catch(() => {}); + const tmpPath = `${destPath}.tmp`; + await RNBlobUtil.fs.unlink(tmpPath).catch(() => {}); + const task = RNBlobUtil.config({ path: tmpPath, fileCache: true }).fetch('GET', url); const onAbort = () => task.cancel(); - cb.signal?.addEventListener('abort', onAbort); + signal?.addEventListener('abort', onAbort); - // DownloadManager reports total as -1 until the size is known; forward the - // received byte count regardless so byte-weighted progress still advances. - task.progress({ count: 100 }, (received, total) => { + task.progress({ count: 50 }, (received, total) => { const recv = Number(received); const tot = Number(total); - cb.onBytes?.(recv, tot > 0 ? tot : recv); + if (recv > 0) onBytes?.(recv, tot > 0 ? tot : 0); }); - try { - await task; - } catch (e) { - await RNBlobUtil.fs.unlink(tmp).catch(() => {}); - throw cb.signal?.aborted ? abortError() : e; - } finally { - cb.signal?.removeEventListener('abort', onAbort); - } - - // DownloadManager doesn't surface an HTTP status; an empty file means failure. - const size = await fileSize(tmp); - if (size <= 0) { - await RNBlobUtil.fs.unlink(tmp).catch(() => {}); - throw RnExecuTorchError('DOWNLOAD_FAILED', `Download of ${url} failed (empty response).`); - } - await RNBlobUtil.fs.mv(tmp, dest); - return dest; -} - -// iOS backend: blob-util streams via the iOS URL session straight to disk. -// Interrupted downloads resume from a `.partial` file via an HTTP Range request. -// `canResume` is set to `false` on an internal retry to avoid recursing forever -// if partial-file assembly ever fails. -async function downloadUrlViaIosStream( - url: string, - dest: string, - cb: DownloadUrlCallbacks, - canResume = true -): Promise { - const part = `${dest}.partial`; - if (!canResume) await RNBlobUtil.fs.unlink(part).catch(() => {}); - const offset = canResume ? await fileSize(part) : 0; - - // Resumed byte ranges land in a separate chunk file that we append onto the - // partial; fresh downloads stream straight into the partial. - const target = offset > 0 ? `${dest}.chunk` : part; - await RNBlobUtil.fs.unlink(target).catch(() => {}); // clear any stale chunk - - const headers: Record = {}; - if (offset > 0) headers.Range = `bytes=${offset}-`; - - if (cb.signal?.aborted) throw abortError(); - - const task = RNBlobUtil.config({ path: target, fileCache: true }).fetch('GET', url, headers); - const onAbort = () => task.cancel(); - cb.signal?.addEventListener('abort', onAbort); - - task.progress({ count: 20 }, (received, total) => { - const recv = Number(received); - const tot = Number(total); - cb.onBytes?.(offset + recv, offset + tot); - }); - - let status: number; try { const res = await task; - status = res.info().status; - } catch (e) { - // Network drop / cancel. Keep the fresh partial for a future resume, but - // discard a resumed chunk — its offset assumptions may not hold. - if (offset > 0) await RNBlobUtil.fs.unlink(target).catch(() => {}); - throw cb.signal?.aborted ? abortError() : e; - } finally { - cb.signal?.removeEventListener('abort', onAbort); - } - - try { - if (status === 416) { - // Range not satisfiable — the partial already holds the whole file. - await RNBlobUtil.fs.unlink(target).catch(() => {}); - } else if (status >= 400) { - await RNBlobUtil.fs.unlink(target).catch(() => {}); + if (res.info().status >= 400 || !(await isDownloaded(tmpPath))) { + await RNBlobUtil.fs.unlink(tmpPath).catch(() => {}); throw RnExecuTorchError( 'DOWNLOAD_FAILED', - `Download of ${url} failed with HTTP status ${status}.` + `Download of ${url} failed with status ${res.info().status}.` ); - } else if (offset > 0) { - if (status === 206) { - // Server honored the range: append the new bytes onto the partial. - await RNBlobUtil.fs.appendFile(part, `file://${target}`, 'uri'); - await RNBlobUtil.fs.unlink(target).catch(() => {}); - } else { - // Server ignored the range (200) and re-sent the whole file: replace. - await RNBlobUtil.fs.unlink(part).catch(() => {}); - await RNBlobUtil.fs.mv(target, part); - } } - await RNBlobUtil.fs.mv(part, dest); - return dest; - } catch (assemblyErr) { - // A `4xx` we deliberately threw must propagate as-is. - if (status >= 400 && status !== 416) throw assemblyErr; - // Assembling the partial failed (e.g. append unsupported). Wipe and retry - // once as a plain full download so correctness never depends on resume. - await RNBlobUtil.fs.unlink(part).catch(() => {}); - await RNBlobUtil.fs.unlink(target).catch(() => {}); - if (canResume) return downloadUrlViaIosStream(url, dest, cb, false); - throw assemblyErr; + await RNBlobUtil.fs.mv(tmpPath, destPath); + return destPath; + } catch (err) { + await RNBlobUtil.fs.unlink(tmpPath).catch(() => {}); + if (signal?.aborted) throw RnExecuTorchError('DOWNLOAD_ABORTED', 'The download was aborted.'); + throw err; + } finally { + signal?.removeEventListener('abort', onAbort); } } -// Plain data containers we recurse into. Anything else (numbers, functions, -// typed arrays, class instances) is left alone. -function isPlainObject(value: unknown): value is Record { - if (typeof value !== 'object' || value === null) return false; - const proto = Object.getPrototypeOf(value); - return proto === Object.prototype || proto === null; -} - -/** - * Walks a value and collects every distinct remote URL sitting on a string - * leaf. Deduplicating here means a URL repeated across fields is fetched once. - * @param node The value to walk. - * @returns The set of remote URLs referenced by `node`. - */ -function collectRemoteSources(node: unknown): Set { - const out = new Set(); - - const visit = (current: unknown): void => { - if (typeof current === 'string') { - if (isRemote(current)) out.add(current); - } else if (Array.isArray(current)) { - for (const item of current) visit(item); - } else if (isPlainObject(current)) { - for (const value of Object.values(current)) visit(value); - } - }; - - visit(node); +// Collect all remote URLs inside a string, array, or nested config object +function collectUrls(node: unknown, out = new Set()): Set { + if (typeof node === 'string') { + if (isRemote(node)) out.add(node); + } else if (Array.isArray(node)) { + for (const item of node) collectUrls(item, out); + } else if (typeof node === 'object' && node !== null) { + for (const val of Object.values(node)) collectUrls(val, out); + } return out; } -/** - * Rebuilds a value with every resolved URL replaced by its local path. - * Branches containing no resolved URL keep their original reference, so an - * untouched config comes back as-is and stays stable across React renders. - * @typeParam T The shape of the value being rewritten. - * @param node The value to rewrite. - * @param resolved Map of remote URL to downloaded local path. - * @returns `node` with resolved URLs swapped for local paths. - */ -function substituteRemoteSources(node: T, resolved: ReadonlyMap): T { - if (typeof node === 'string') { - return (resolved.get(node) ?? node) as T; - } - if (Array.isArray(node)) { - let changed = false; - const next = node.map((item) => { - const mapped = substituteRemoteSources(item, resolved); - changed ||= mapped !== item; - return mapped; - }); - return (changed ? next : node) as T; - } - if (isPlainObject(node)) { - let changed = false; +// Replace remote URLs in config object with local downloaded file paths +function replaceUrls(node: T, resolved: Map): T { + if (typeof node === 'string') return (resolved.get(node) ?? node) as T; + if (Array.isArray(node)) return node.map((item) => replaceUrls(item, resolved)) as T; + if (typeof node === 'object' && node !== null) { const next: Record = {}; - for (const [key, value] of Object.entries(node)) { - const mapped = substituteRemoteSources(value, resolved); - changed ||= mapped !== value; - next[key] = mapped; - } - return (changed ? next : node) as T; + for (const [k, v] of Object.entries(node)) next[k] = replaceUrls(v, resolved); + return next as T; } return node; } @@ -392,23 +119,6 @@ function substituteRemoteSources(node: T, resolved: ReadonlyMap` factory: - * - * ```ts - * const model = await download(models.classification.EFFICIENTNET_V2_S.XNNPACK_FP32); - * const { classify, dispose } = await createClassifier(model); - * ``` - * - * Downloads go to a persistent cache: on Android via the system DownloadManager - * (handles multi-GB files and continues in the background), on iOS via a - * streaming request that resumes an interrupted download from where it stopped. - * When a config references several files, overall progress is weighted by their - * byte sizes so a large model isn't reported the same as a tiny tokenizer. * @category Utils * @typeParam T The shape of the value being resolved. * @param source A URL, a local path, or any nested object/array holding them. @@ -416,44 +126,71 @@ function substituteRemoteSources(node: T, resolved: ReadonlyMap(source: T, options: DownloadOptions = {}): Promise { - const urls = [...collectRemoteSources(source)]; - - // Nothing to fetch — every source is already a local path. + const urls = [...collectUrls(source)]; if (urls.length === 0) { options.onProgress?.(1); return source; } - // Weight overall progress by real byte sizes so a 1 GB model isn't treated - // like a 1 MB tokenizer. When any HEAD fails we fall back to equal weighting. - const sizes = await Promise.all(urls.map(remoteSize)); - const haveAllSizes = sizes.every((size) => size > 0); - const total = haveAllSizes ? sizes.reduce((a, b) => a + b, 0) : urls.length; - + const resolved = new Map(); const received = new Array(urls.length).fill(0); - const report = () => { + const totals = new Array(urls.length).fill(0); + + // Fast-path: pre-fill cached files + for (let i = 0; i < urls.length; i++) { + const url = urls[i]!; + const destPath = cachePathFor(url); + if (!options.forceDownload && (await isDownloaded(destPath))) { + const stat = await RNBlobUtil.fs.stat(destPath); + const sz = Number(stat.size); + received[i] = sz; + totals[i] = sz; + resolved.set(url, destPath); + } + } + + // If all files are cached, return immediately with 100% progress + if (resolved.size === urls.length) { + options.onProgress?.(1); + return replaceUrls(source, resolved); + } + + const updateProgress = () => { if (!options.onProgress) return; - const sum = received.reduce((a, b) => a + b, 0); - options.onProgress(total > 0 ? Math.min(sum / total, 1) : 0); + const sumReceived = received.reduce((a, b) => a + b, 0); + const sumTotals = totals.reduce((a, b) => a + b, 0); + options.onProgress(sumTotals > 0 ? Math.min(sumReceived / sumTotals, 1) : 0); }; - const resolved = new Map(); + // Initial progress update for any pre-cached files in bundle + updateProgress(); + + // Download uncached files concurrently await Promise.all( - urls.map((url, i) => - // Telemetry fires inside downloadUrl, only for genuine (non-cached) fetches. - downloadUrl(url, { - signal: options.signal, - forceDownload: options.forceDownload, - onBytes: (recv, tot) => { - received[i] = haveAllSizes ? recv : tot > 0 ? recv / tot : 0; - report(); + urls.map(async (url, i) => { + if (resolved.has(url)) return; + + const destPath = cachePathFor(url); + const downloadedPath = await downloadFile( + url, + destPath, + (recv, tot) => { + received[i] = recv; + if (tot > 0) totals[i] = tot; + updateProgress(); }, - }).then((path) => { - resolved.set(url, path); - }) - ) + options.signal + ); + + const stat = await RNBlobUtil.fs.stat(downloadedPath); + const finalSize = Number(stat.size); + received[i] = finalSize; + totals[i] = finalSize; + resolved.set(url, downloadedPath); + updateProgress(); + }) ); options.onProgress?.(1); - return substituteRemoteSources(source, resolved); + return replaceUrls(source, resolved); } diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts new file mode 100644 index 0000000000..5d3bde5176 --- /dev/null +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -0,0 +1,40 @@ +import { useModel } from './useModel'; +import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; +import { + createLLMChatSession, + type LLMModel, + type LLMChatSessionOptions, +} from '../extensions/llm/tasks/llmChatSession'; + +/** + * React hook to load and run an LLM chat session model. + * + * This hook manages downloading (if they are remote URLs) and loading the `.pte` model + * file, `tokenizer.json`, and `tokenizer_config.json`, tracking download progress and errors, + * and cleaning up native memory when the component unmounts or configuration changes. + * @category Hooks + * @param config The LLM model configuration. + * @param options Chat session options and load/caching options. See {@link ResourceOptions}. + * @returns An object containing the session's loading state, error, download progress, + * and chat functions. + */ +export function useLLMChatSession( + config: LLMModel, + options?: LLMChatSessionOptions & ResourceOptions +) { + const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); + const { model: session, error } = useModel( + (res) => createLLMChatSession(res, options), + resource ?? null + ); + + return { + isReady: !!session, + error: downloadError || error, + downloadProgress, + resource, + sendMessage: session?.sendMessage, + getHistory: session?.getHistory, + stop: session?.stop, + }; +} diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index 0865295145..d7cb910926 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -6,6 +6,7 @@ export * from './hooks/useInstanceSegmenter'; export * from './hooks/useKeypointDetector'; export * from './hooks/useObjectDetector'; export * from './hooks/useTokenizer'; +export * from './hooks/useLLMChatSession'; export * from './hooks/useTextEmbedder'; export * from './hooks/usePrivacyFilter'; export * from './hooks/useImageEmbedder'; @@ -32,6 +33,7 @@ export * from './extensions/cv/tasks/keypointDetection'; export * from './extensions/cv/tasks/objectDetection'; export * from './extensions/cv/tasks/imageEmbedding'; export * from './extensions/cv/tasks/sdxsTextToImage'; +export * from './extensions/llm/tasks/llmChatSession'; export * from './extensions/nlp/tasks/tokenization'; export * from './extensions/nlp/tasks/textEmbedding'; export * from './extensions/nlp/tasks/privacyFilter'; @@ -50,6 +52,7 @@ export * as schema from './core/schema'; export * as math from './extensions/math'; export * as cv from './extensions/cv'; +export * as llm from './extensions/llm'; export * as nlp from './extensions/nlp'; export * as speech from './extensions/speech'; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index fe9155015e..5a1866c50e 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -14,6 +14,7 @@ import { type WhisperSttModel, WHISPER_LANGUAGES, } from './extensions/speech/tasks/whisperSpeechToText'; +import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -816,6 +817,269 @@ const PRIVACY_FILTER_NEMOTRON_MLX_INT8: PrivacyFilterModel', end: '<|image_end|>' }, + targetShape: [3, 512, 512] as const, + preprocessorOpts: { + resizeMode: 'letterbox' as const, + interpolation: 'linear' as const, + normalizeOpts: { alpha: 1.0, beta: 0.0 }, + }, + }, +}; +const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; +const LFM2_5_VL_450M_MLX_INT4: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; +const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, +}; + +const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; + +const BIELIK_V3_1_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${BIELIK_V3_1_5B_BASE_URL}/xnnpack/bielik_v3_0_1_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer_config.json`, +}; +const BIELIK_V3_1_5B_XNNPACK_FP16: LLMModel = { + modelPath: `${BIELIK_V3_1_5B_BASE_URL}/xnnpack/bielik_v3_0_1_5b_xnnpack_fp16.pte`, + tokenizerPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${BIELIK_V3_1_5B_BASE_URL}/tokenizer_config.json`, +}; + +const LLAMA3_2_BASE_URL = `${BASE_URL}-llama-3.2/${VERSION_TAG}`; + +const LLAMA3_2_3B_SPINQUANT: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/3b/xnnpack/llama_3_2_3b_xnnpack_spinquant.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; +const LLAMA3_2_3B_BF16: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/3b/xnnpack/llama_3_2_3b_xnnpack_bf16.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; +const LLAMA3_2_1B_SPINQUANT: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/1b/xnnpack/llama_3_2_1b_xnnpack_spinquant.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; +const LLAMA3_2_1B_BF16: LLMModel = { + modelPath: `${LLAMA3_2_BASE_URL}/1b/xnnpack/llama_3_2_1b_xnnpack_bf16.pte`, + tokenizerPath: `${LLAMA3_2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${LLAMA3_2_BASE_URL}/tokenizer_config.json`, +}; + +const SMOLLM2_BASE_URL = `${BASE_URL}-smolLm-2/${VERSION_TAG}`; + +const SMOLLM2_135M_8DA4W: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/135m/xnnpack/smollm2_135m_xnnpack_8da4w.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_135M_BF16: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/135m/xnnpack/smollm2_135m_xnnpack_bf16.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_360M_8DA4W: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/360m/xnnpack/smollm2_360m_xnnpack_8da4w.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_360M_BF16: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/360m/xnnpack/smollm2_360m_xnnpack_bf16.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_1_7B_8DA4W: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/1_7b/xnnpack/smollm2_1_7b_xnnpack_8da4w.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; +const SMOLLM2_1_7B_BF16: LLMModel = { + modelPath: `${SMOLLM2_BASE_URL}/1_7b/xnnpack/smollm2_1_7b_xnnpack_bf16.pte`, + tokenizerPath: `${SMOLLM2_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${SMOLLM2_BASE_URL}/tokenizer_config.json`, +}; + +const HAMMER2_1_BASE_URL = `${BASE_URL}-hammer-2.1/${VERSION_TAG}`; + +const HAMMER2_1_0_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_0_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/0_5b/xnnpack/hammer_2_1_0_5b_xnnpack_bf16.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_1_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_1_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/1_5b/xnnpack/hammer_2_1_1_5b_xnnpack_bf16.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_3B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/3b/xnnpack/hammer_2_1_3b_xnnpack_8da4w.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; +const HAMMER2_1_3B_XNNPACK_BF16: LLMModel = { + modelPath: `${HAMMER2_1_BASE_URL}/3b/xnnpack/hammer_2_1_3b_xnnpack_bf16.pte`, + tokenizerPath: `${HAMMER2_1_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${HAMMER2_1_BASE_URL}/tokenizer_config.json`, +}; + +const PHI4_MINI_BASE_URL = `${BASE_URL}-phi-4-mini/${VERSION_TAG}`; + +const PHI4_MINI_XNNPACK_8DA4W: LLMModel = { + modelPath: `${PHI4_MINI_BASE_URL}/xnnpack/phi_4_mini_xnnpack_8da4w.pte`, + tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, +}; +const PHI4_MINI_XNNPACK_BF16: LLMModel = { + modelPath: `${PHI4_MINI_BASE_URL}/xnnpack/phi_4_mini_xnnpack_bf16.pte`, + tokenizerPath: `${PHI4_MINI_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${PHI4_MINI_BASE_URL}/tokenizer_config.json`, +}; + +const QWEN2_5_BASE_URL = `${BASE_URL}-qwen-2.5/${VERSION_TAG}`; + +const QWEN2_5_0_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_0_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/0_5b/xnnpack/qwen_2_5_0_5b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_1_5B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_1_5B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/1_5b/xnnpack/qwen_2_5_1_5b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_3B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/3b/xnnpack/qwen_2_5_3b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; +const QWEN2_5_3B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN2_5_BASE_URL}/3b/xnnpack/qwen_2_5_3b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN2_5_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, +}; + +const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; + +const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; +const GEMMA4_E2B_MLX_INT4: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; + +const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; + +const QWEN3_0_6B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/0_6b/xnnpack/qwen_3_0_6b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_0_6B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/0_6b/xnnpack/qwen_3_0_6b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_1_7B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/1_7b/xnnpack/qwen_3_1_7b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_1_7B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/1_7b/xnnpack/qwen_3_1_7b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_4B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/4b/xnnpack/qwen_3_4b_xnnpack_8da4w.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; +const QWEN3_4B_XNNPACK_BF16: LLMModel = { + modelPath: `${QWEN3_BASE_URL}/4b/xnnpack/qwen_3_4b_xnnpack_bf16.pte`, + tokenizerPath: `${QWEN3_BASE_URL}/tokenizer.json`, + tokenizerConfigPath: `${QWEN3_BASE_URL}/tokenizer_config.json`, +}; + /** * Registry of pre-configured ExecuTorch models. * @@ -1281,6 +1545,203 @@ export const models = { ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER, }, + /** + * Generative Large Language Models (LLMs) for instruction following, + * chat, text generation, and reasoning. + */ + llm: { + /** + * Liquid AI LFM 2.5 1.2B general-purpose text model. Excellent for complex + * on-device reasoning, instruction following, and fast multi-turn chat. + */ + LFM2_5_1_2B: { + ...LFM2_5_1_2B_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_1_2B_XNNPACK_8DA4W, + XNNPACK_FP16: LFM2_5_1_2B_XNNPACK_FP16, + MLX_INT4: LFM2_5_1_2B_MLX_INT4, + }, + /** + * Liquid AI LFM 2.5 350M ultra-compact text model. Best for low-latency text + * completion, quick responses, and resource-constrained devices. + */ + LFM2_5_350M: { + ...LFM2_5_350M_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_350M_XNNPACK_8DA4W, + XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, + MLX_INT4: LFM2_5_350M_MLX_INT4, + }, + /** + * Liquid AI LFM 2.5 450M vision-language model. Optimized for real-time + * visual QA, image description, and low-latency multimodal chat. + */ + LFM2_5_VL_450M: { + ...LFM2_5_VL_450M_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, + MLX_INT4: LFM2_5_VL_450M_MLX_INT4, + }, + /** + * Liquid AI LFM 2.5 1.6B vision-language model. Higher quality visual + * understanding, detailed image analysis, and complex multimodal tasks. + */ + LFM2_5_VL_1_6B: { + ...LFM2_5_VL_1_6B_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, + }, + /** + * Bielik v3 1.5B Polish & English language model. Fine-tuned specifically for + * native Polish fluency, grammar, and bilingual translation. + */ + BIELIK_V3_1_5B: { + ...BIELIK_V3_1_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: BIELIK_V3_1_5B_XNNPACK_8DA4W, + XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, + }, + /** + * Meta Llama 3.2 1B multilingual text model. Ideal for lightweight mobile + * chat, summary generation, and multilingual prompt processing. + */ + LLAMA3_2_1B: { + ...LLAMA3_2_1B_SPINQUANT, + XNNPACK_SPINQUANT: LLAMA3_2_1B_SPINQUANT, + XNNPACK_BF16: LLAMA3_2_1B_BF16, + }, + /** + * Meta Llama 3.2 3B multilingual text model. Strong instruction following, + * detailed content creation, and high-precision text reasoning. + */ + LLAMA3_2_3B: { + ...LLAMA3_2_3B_SPINQUANT, + XNNPACK_SPINQUANT: LLAMA3_2_3B_SPINQUANT, + XNNPACK_BF16: LLAMA3_2_3B_BF16, + }, + /** + * Hugging Face SmolLM2 135M sub-parameter model. Best for micro-footprint + * background tasks, simple text tagging, and instant autocomplete. + */ + SMOLLM2_135M: { + ...SMOLLM2_135M_8DA4W, + XNNPACK_8DA4W: SMOLLM2_135M_8DA4W, + XNNPACK_BF16: SMOLLM2_135M_BF16, + }, + /** + * Hugging Face SmolLM2 360M compact model. Balanced speed and intelligence + * for lightweight conversational assistants. + */ + SMOLLM2_360M: { + ...SMOLLM2_360M_8DA4W, + XNNPACK_8DA4W: SMOLLM2_360M_8DA4W, + XNNPACK_BF16: SMOLLM2_360M_BF16, + }, + /** + * Hugging Face SmolLM2 1.7B language model. Powerful general-purpose text + * generation, creative writing, and general knowledge Q&A. + */ + SMOLLM2_1_7B: { + ...SMOLLM2_1_7B_8DA4W, + XNNPACK_8DA4W: SMOLLM2_1_7B_8DA4W, + XNNPACK_BF16: SMOLLM2_1_7B_BF16, + }, + /** + * Hammer 2.1 0.5B function-calling model. Specialized for lightweight agentic + * tool calling, JSON extraction, and structured output parsing. + */ + HAMMER2_1_0_5B: { + ...HAMMER2_1_0_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_0_5B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_0_5B_XNNPACK_BF16, + }, + /** + * Hammer 2.1 1.5B function-calling model. Optimized for multi-tool agentic + * workflows, API function calling, and structured JSON schemas. + */ + HAMMER2_1_1_5B: { + ...HAMMER2_1_1_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_1_5B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_1_5B_XNNPACK_BF16, + }, + /** + * Hammer 2.1 3B function-calling model. High-capacity agentic reasoning, + * complex multi-step tool execution, and robust schema compliance. + */ + HAMMER2_1_3B: { + ...HAMMER2_1_3B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_3B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, + }, + /** + * Microsoft Phi-4 Mini 3.8B reasoning model. Exceptional for math problem + * solving, logical reasoning, code synthesis, and analytical tasks. + */ + PHI4_MINI: { + ...PHI4_MINI_XNNPACK_8DA4W, + XNNPACK_8DA4W: PHI4_MINI_XNNPACK_8DA4W, + XNNPACK_BF16: PHI4_MINI_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 2.5 0.5B multilingual model. Extremely efficient for fast + * multi-language translation and basic conversational chat. + */ + QWEN2_5_0_5B: { + ...QWEN2_5_0_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_0_5B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_0_5B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 2.5 1.5B multilingual model. Great for balanced multilingual + * chat, text summarization, and cross-lingual understanding. + */ + QWEN2_5_1_5B: { + ...QWEN2_5_1_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_1_5B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_1_5B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 2.5 3B multilingual model. High capability across 29+ + * languages for complex translation, long-form writing, and Q&A. + */ + QWEN2_5_3B: { + ...QWEN2_5_3B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_3B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 3 0.6B next-gen text model. Low-latency multilingual model + * for fast turn-taking and concise response generation. + */ + QWEN3_0_6B: { + ...QWEN3_0_6B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_0_6B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_0_6B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 3 1.7B next-gen text model. Versatile multilingual assistant + * for high-quality instruction following and knowledge retrieval. + */ + QWEN3_1_7B: { + ...QWEN3_1_7B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_1_7B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_1_7B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 3 4B high-capacity text model. Top-tier multilingual + * reasoning, technical content generation, and multi-turn dialogue. + */ + QWEN3_4B: { + ...QWEN3_4B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_4B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, + }, + /** + * Google Gemma 4 E2B generative text model. Built on Google's Gemini tech + * for high-fidelity instruction following and mobile assistance. + */ + GEMMA4_E2B: { + ...GEMMA4_E2B_XNNPACK_8DA4W, + XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, + MLX_INT4: GEMMA4_E2B_MLX_INT4, + }, + }, + /** * Text embedding models mapping sentences and documents into dense vector * representations for semantic search and RAG. diff --git a/yarn.lock b/yarn.lock index 875b8db572..e45de3eec1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3130,6 +3130,13 @@ __metadata: languageName: node linkType: hard +"@huggingface/jinja@npm:^0.5.9": + version: 0.5.9 + resolution: "@huggingface/jinja@npm:0.5.9" + checksum: 10/8147f05df29b609ebb923f9e294f485897c5b5b92cddb1e41d8469b2992def242d9b536b8e9ca450a24a88fae0df6764986ab1248033f34edd4f5eb9dc2c4d78 + languageName: node + linkType: hard + "@humanwhocodes/config-array@npm:^0.13.0": version: 0.13.0 resolution: "@humanwhocodes/config-array@npm:0.13.0" @@ -11412,11 +11419,14 @@ __metadata: "@react-native/metro-config": "npm:^0.86.0" "@react-navigation/drawer": "npm:^7.9.4" "@react-navigation/native": "npm:^7.2.2" + "@shopify/react-native-skia": "npm:2.6.2" "@types/react": "npm:~19.2.0" babel-preset-expo: "npm:~56.0.14" expo: "npm:~56.0.9" expo-build-properties: "npm:~56.0.17" expo-constants: "npm:~56.0.17" + expo-image-manipulator: "npm:~56.0.18" + expo-image-picker: "npm:~56.0.18" expo-linking: "npm:~56.0.13" expo-router: "npm:~56.2.9" react: "npm:19.2.3" @@ -12398,6 +12408,7 @@ __metadata: resolution: "react-native-executorch@workspace:packages/react-native-executorch" dependencies: "@babel/core": "npm:^7.25.1" + "@huggingface/jinja": "npm:^0.5.9" "@react-native/babel-preset": "npm:0.83.6" "@react-native/metro-config": "npm:^0.86.0" "@types/react": "npm:^19.1.12"