Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .clang-format
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# clang-format configuration for the react-native-executorch C/C++/Objective-C
# sources. Based on the LLVM style (the project's prior implicit default) with a
# wider column limit so clang-format stops aggressively wrapping function
# signatures, chained ternaries and string concatenations. See issue #1217.
#
# Vendored libraries (common/ada, common/pfft, common/runner) and the generated
# ErrorCodes.h are excluded from formatting via .clang-format-ignore.
BasedOnStyle: LLVM
Standard: c++20
ColumnLimit: 100
InsertNewlineAtEOF: true
12 changes: 12 additions & 0 deletions .clang-format-ignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Files clang-format must skip. Patterns are gitignore-style, relative to this
# file. This keeps `clang-format -i` (and the pre-commit hook) from reformatting
# vendored third-party code and generated sources.

# Vendored libraries that live outside third-party/.
packages/react-native-executorch/common/ada/**
packages/react-native-executorch/common/pfft/**
packages/react-native-executorch/common/runner/**

# Generated verbatim by `yarn codegen:errors` and verified in CI with a raw
# diff; reformatting would rewrap its comments and break that check.
packages/react-native-executorch/common/rnexecutorch/ErrorCodes.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ cv::Mat unrotateMat(const cv::Mat &src, int32_t rotation) {

extern "C" {

JNIEXPORT jlong JNICALL
Java_com_executorch_webrtc_ExecutorchFrameProcessor_loadModel(
JNIEXPORT jlong JNICALL Java_com_executorch_webrtc_ExecutorchFrameProcessor_loadModel(
JNIEnv *env, jobject thiz, jstring modelPath) {
const char *pathChars = env->GetStringUTFChars(modelPath, nullptr);
if (pathChars == nullptr) {
Expand All @@ -82,8 +81,8 @@ Java_com_executorch_webrtc_ExecutorchFrameProcessor_loadModel(
std::vector<std::string> allClasses = {"foreground", "background"};

auto handle = std::make_unique<ProcessorHandle>();
handle->segmentation = std::make_unique<BaseSemanticSegmentation>(
path, normMean, normStd, allClasses, nullptr);
handle->segmentation =
std::make_unique<BaseSemanticSegmentation>(path, normMean, normStd, allClasses, nullptr);

auto inputShapes = handle->segmentation->getAllInputShapes();
if (!inputShapes.empty() && inputShapes[0].size() >= 4) {
Expand All @@ -99,10 +98,9 @@ Java_com_executorch_webrtc_ExecutorchFrameProcessor_loadModel(
}
}

JNIEXPORT jbyteArray JNICALL
Java_com_executorch_webrtc_ExecutorchFrameProcessor_runSegmentation(
JNIEnv *env, jobject thiz, jlong handlePtr, jbyteArray rgbaData, jint width,
jint height, jint rotation) {
JNIEXPORT jbyteArray JNICALL Java_com_executorch_webrtc_ExecutorchFrameProcessor_runSegmentation(
JNIEnv *env, jobject thiz, jlong handlePtr, jbyteArray rgbaData, jint width, jint height,
jint rotation) {
if (handlePtr == 0) {
return nullptr;
}
Expand All @@ -127,15 +125,13 @@ Java_com_executorch_webrtc_ExecutorchFrameProcessor_runSegmentation(
pixelData.scalarType = executorch::aten::ScalarType::Byte;

std::set<std::string, std::less<>> classesOfInterest = {"foreground"};
auto result = handle->segmentation->generateFromPixels(
pixelData, classesOfInterest, false);
auto result = handle->segmentation->generateFromPixels(pixelData, classesOfInterest, false);

cv::Mat mask;
if (result.classBuffers && result.classBuffers->count("foreground")) {
auto &fgBuffer = result.classBuffers->at("foreground");
auto *fgData = reinterpret_cast<float *>(fgBuffer->data());
mask = cv::Mat(handle->modelHeight, handle->modelWidth, CV_32FC1, fgData)
.clone();
mask = cv::Mat(handle->modelHeight, handle->modelWidth, CV_32FC1, fgData).clone();
} else {
LOGE("No foreground mask in result");
env->ReleaseByteArrayElements(rgbaData, rgbaPtr, JNI_ABORT);
Expand All @@ -151,16 +147,15 @@ Java_com_executorch_webrtc_ExecutorchFrameProcessor_runSegmentation(
handle->previousMask = mask.clone();
handle->hasHistory = true;
} else {
cv::addWeighted(handle->previousMask, EMA_ALPHA, mask, 1.0f - EMA_ALPHA,
0, handle->previousMask);
cv::addWeighted(handle->previousMask, EMA_ALPHA, mask, 1.0f - EMA_ALPHA, 0,
handle->previousMask);
mask = handle->previousMask.clone();
}

cv::Mat maskRotated = unrotateMat(mask, rotation);

cv::Mat maskResized;
cv::resize(maskRotated, maskResized, cv::Size(width, height), 0, 0,
cv::INTER_LINEAR);
cv::resize(maskRotated, maskResized, cv::Size(width, height), 0, 0, cv::INTER_LINEAR);

cv::Mat maskBytes;
maskResized.convertTo(maskBytes, CV_8UC1, 255.0);
Expand All @@ -171,8 +166,7 @@ Java_com_executorch_webrtc_ExecutorchFrameProcessor_runSegmentation(
env->ReleaseByteArrayElements(rgbaData, rgbaPtr, JNI_ABORT);
return nullptr;
}
env->SetByteArrayRegion(output, 0, maskSize,
reinterpret_cast<jbyte *>(maskBytes.data));
env->SetByteArrayRegion(output, 0, maskSize, reinterpret_cast<jbyte *>(maskBytes.data));
env->ReleaseByteArrayElements(rgbaData, rgbaPtr, JNI_ABORT);
return output;

Expand All @@ -183,8 +177,7 @@ Java_com_executorch_webrtc_ExecutorchFrameProcessor_runSegmentation(
}
}

JNIEXPORT void JNICALL
Java_com_executorch_webrtc_ExecutorchFrameProcessor_unloadModel(
JNIEXPORT void JNICALL Java_com_executorch_webrtc_ExecutorchFrameProcessor_unloadModel(
JNIEnv *env, jobject thiz, jlong handlePtr) {
if (handlePtr == 0) {
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,8 @@ - (void)configureWithModelPath:(NSString *)modelPath {
std::vector<float> normMean = {};
std::vector<float> normStd = {};
std::vector<std::string> allClasses = {"foreground", "background"};
seg = std::make_unique<BaseSemanticSegmentation>(
std::string([modelPath UTF8String]), normMean, normStd, allClasses,
nullptr);
seg = std::make_unique<BaseSemanticSegmentation>(std::string([modelPath UTF8String]), normMean,
normStd, allClasses, nullptr);
auto inputShapes = seg->getAllInputShapes();
if (!inputShapes.empty() && inputShapes[0].size() >= 4) {
h = inputShapes[0][inputShapes[0].size() - 2];
Expand Down Expand Up @@ -181,8 +180,8 @@ - (RTCVideoFrame *)processFrameLocked:(RTCVideoFrame *)frame {
}

CVPixelBufferRef outputBuffer = NULL;
CVReturn poolStatus = CVPixelBufferPoolCreatePixelBuffer(
kCFAllocatorDefault, _outputPool, &outputBuffer);
CVReturn poolStatus =
CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, _outputPool, &outputBuffer);
if (poolStatus != kCVReturnSuccess) {
if (ownsInput) {
CVPixelBufferRelease(inputPixelBuffer);
Expand All @@ -193,15 +192,13 @@ - (RTCVideoFrame *)processFrameLocked:(RTCVideoFrame *)frame {
CIImage *original = [CIImage imageWithCVPixelBuffer:inputPixelBuffer];
CGFloat scaleX = (CGFloat)width / maskImage.extent.size.width;
CGFloat scaleY = (CGFloat)height / maskImage.extent.size.height;
CIImage *scaledMask = [maskImage
imageByApplyingTransform:CGAffineTransformMakeScale(scaleX, scaleY)];
CIImage *scaledMask =
[maskImage imageByApplyingTransform:CGAffineTransformMakeScale(scaleX, scaleY)];

CIFilter *blurFilter = [CIFilter filterWithName:@"CIGaussianBlur"];
[blurFilter setValue:[original imageByClampingToExtent]
forKey:kCIInputImageKey];
[blurFilter setValue:[original imageByClampingToExtent] forKey:kCIInputImageKey];
[blurFilter setValue:@(_blurRadius) forKey:kCIInputRadiusKey];
CIImage *blurred =
[blurFilter.outputImage imageByCroppingToRect:original.extent];
CIImage *blurred = [blurFilter.outputImage imageByCroppingToRect:original.extent];

CIFilter *blendFilter = [CIFilter filterWithName:@"CIBlendWithMask"];
[blendFilter setValue:original forKey:kCIInputImageKey];
Expand All @@ -216,12 +213,10 @@ - (RTCVideoFrame *)processFrameLocked:(RTCVideoFrame *)frame {
colorSpace:colorSpace];
CGColorSpaceRelease(colorSpace);

RTCCVPixelBuffer *rtcBuffer =
[[RTCCVPixelBuffer alloc] initWithPixelBuffer:outputBuffer];
RTCVideoFrame *outputFrame =
[[RTCVideoFrame alloc] initWithBuffer:rtcBuffer
rotation:frame.rotation
timeStampNs:frame.timeStampNs];
RTCCVPixelBuffer *rtcBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:outputBuffer];
RTCVideoFrame *outputFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcBuffer
rotation:frame.rotation
timeStampNs:frame.timeStampNs];

CVPixelBufferRelease(outputBuffer);
if (ownsInput) {
Expand Down Expand Up @@ -271,10 +266,9 @@ - (CIImage *)generateMaskLockedForPixelBuffer:(CVPixelBufferRef)pixelBuffer
yMat.copyTo(yuvMat(cv::Rect(0, 0, (int)width, (int)height)));
std::vector<cv::Mat> uvChannels;
cv::split(uvMat, uvChannels);
uvChannels[0].copyTo(
yuvMat(cv::Rect(0, (int)height, (int)width / 2, (int)height / 2)));
uvChannels[1].copyTo(yuvMat(cv::Rect((int)width / 2, (int)height,
(int)width / 2, (int)height / 2)));
uvChannels[0].copyTo(yuvMat(cv::Rect(0, (int)height, (int)width / 2, (int)height / 2)));
uvChannels[1].copyTo(
yuvMat(cv::Rect((int)width / 2, (int)height, (int)width / 2, (int)height / 2)));
cv::cvtColor(yuvMat, rgbMat, cv::COLOR_YUV2RGB_I420);
} else {
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
Expand Down Expand Up @@ -302,8 +296,7 @@ - (CIImage *)generateMaskLockedForPixelBuffer:(CVPixelBufferRef)pixelBuffer
pixelData.scalarType = executorch::aten::ScalarType::Byte;

std::set<std::string, std::less<>> classesOfInterest = {"foreground"};
auto result =
_segmentation->generateFromPixels(pixelData, classesOfInterest, false);
auto result = _segmentation->generateFromPixels(pixelData, classesOfInterest, false);

if (result.classBuffers && result.classBuffers->count("foreground")) {
auto &fgBuffer = result.classBuffers->at("foreground");
Expand Down Expand Up @@ -331,8 +324,7 @@ - (CIImage *)generateMaskLockedForPixelBuffer:(CVPixelBufferRef)pixelBuffer
if (_previousMask.empty() || _previousMask.size() != maskRotated.size()) {
_previousMask = maskRotated.clone();
} else {
cv::addWeighted(maskRotated, EMA_ALPHA, _previousMask, 1.0f - EMA_ALPHA, 0,
maskRotated);
cv::addWeighted(maskRotated, EMA_ALPHA, _previousMask, 1.0f - EMA_ALPHA, 0, maskRotated);
_previousMask = maskRotated.clone();
}

Expand Down Expand Up @@ -380,9 +372,9 @@ - (CVPixelBufferRef)createPixelBufferFromI420:(id<RTCVideoFrameBuffer>)buffer {
(id)kCVPixelBufferIOSurfacePropertiesKey : @{},
};
CVPixelBufferRef pixelBuffer = NULL;
CVReturn result = CVPixelBufferCreate(
kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA,
(__bridge CFDictionaryRef)attrs, &pixelBuffer);
CVReturn result =
CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA,
(__bridge CFDictionaryRef)attrs, &pixelBuffer);
if (result != kCVReturnSuccess) {
return NULL;
}
Expand All @@ -398,8 +390,7 @@ - (CVPixelBufferRef)createPixelBufferFromI420:(id<RTCVideoFrameBuffer>)buffer {
memcpy(uvDst + row * uvWidth, dataU + row * strideU, uvWidth);
}
for (int row = 0; row < uvHeight; row++) {
memcpy(uvDst + uvHeight * uvWidth + row * uvWidth, dataV + row * strideV,
uvWidth);
memcpy(uvDst + uvHeight * uvWidth + row * uvWidth, dataV + row * strideV, uvWidth);
}

cv::Mat bgraMat;
Expand All @@ -425,9 +416,9 @@ - (CVPixelBufferRef)createGrayscalePixelBuffer:(cv::Mat &)grayMat {
(id)kCVPixelBufferIOSurfacePropertiesKey : @{},
};
CVPixelBufferRef pixelBuffer = NULL;
CVReturn result = CVPixelBufferCreate(
kCFAllocatorDefault, width, height, kCVPixelFormatType_OneComponent8,
(__bridge CFDictionaryRef)attrs, &pixelBuffer);
CVReturn result =
CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_OneComponent8,
(__bridge CFDictionaryRef)attrs, &pixelBuffer);
if (result != kCVReturnSuccess) {
return NULL;
}
Expand Down Expand Up @@ -457,8 +448,7 @@ - (void)ensurePoolLockedForWidth:(size_t)width height:(size_t)height {
(id)kCVPixelBufferIOSurfacePropertiesKey : @{},
(id)kCVPixelBufferMetalCompatibilityKey : @YES,
};
CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL,
(__bridge CFDictionaryRef)attrs, &_outputPool);
CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL, (__bridge CFDictionaryRef)attrs, &_outputPool);
_poolWidth = width;
_poolHeight = height;
}
Expand Down
11 changes: 3 additions & 8 deletions packages/react-native-executorch-webrtc/ios/ExecutorchWebRTC.mm
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,11 @@ + (void)ensureRegistered {
[_processor unloadModel];
}

RCT_EXPORT_METHOD(setBlurRadius : (double)radius) {
[_processor setBlurRadius:(float)radius];
}
RCT_EXPORT_METHOD(setBlurRadius : (double)radius) { [_processor setBlurRadius:(float)radius]; }

RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(isAvailable) { return @YES; }

RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(getProcessorName) {
return PROCESSOR_NAME;
}
RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(getProcessorName) { return PROCESSOR_NAME; }

#pragma mark - Legacy API (for backward compatibility)

Expand All @@ -54,8 +50,7 @@ + (void)ensureRegistered {
[self initialize:modelPath];
}

RCT_EXPORT_METHOD(configureBackgroundBlur : (NSString *)
modelPath blurIntensity : (int)intensity) {
RCT_EXPORT_METHOD(configureBackgroundBlur : (NSString *)modelPath blurIntensity : (int)intensity) {
[self initialize:modelPath];
[self setBlurRadius:intensity];
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,13 @@ JavaVM *java_machine;
using namespace facebook::jni;

ETInstallerModule::ETInstallerModule(
jni::alias_ref<ETInstallerModule::jhybridobject> &jThis,
jsi::Runtime *jsiRuntime,
jni::alias_ref<ETInstallerModule::jhybridobject> &jThis, jsi::Runtime *jsiRuntime,
const std::shared_ptr<facebook::react::CallInvoker> &jsCallInvoker)
: javaPart_(make_global(jThis)), jsiRuntime_(jsiRuntime),
jsCallInvoker_(jsCallInvoker) {}
: javaPart_(make_global(jThis)), jsiRuntime_(jsiRuntime), jsCallInvoker_(jsCallInvoker) {}

jni::local_ref<ETInstallerModule::jhybriddata> ETInstallerModule::initHybrid(
jni::alias_ref<jhybridobject> jThis, jlong jsContext,
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>
jsCallInvokerHolder) {
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder) {
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
auto rnRuntime = reinterpret_cast<jsi::Runtime *>(jsContext);
return makeCxxInstance(jThis, rnRuntime, jsCallInvoker);
Expand All @@ -30,8 +27,7 @@ jni::local_ref<ETInstallerModule::jhybriddata> ETInstallerModule::initHybrid(
void ETInstallerModule::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", ETInstallerModule::initHybrid),
makeNativeMethod("injectJSIBindings",
ETInstallerModule::injectJSIBindings),
makeNativeMethod("injectJSIBindings", ETInstallerModule::injectJSIBindings),
});
}

Expand All @@ -48,12 +44,11 @@ void ETInstallerModule::injectJSIBindings() {
}
}
static jclass cls = javaClassStatic().get();
static jmethodID method = env->GetStaticMethodID(
cls, "fetchByteDataFromUrl", "(Ljava/lang/String;)[B");
static jmethodID method =
env->GetStaticMethodID(cls, "fetchByteDataFromUrl", "(Ljava/lang/String;)[B");

jstring jUrl = env->NewStringUTF(url.c_str());
jbyteArray byteData =
(jbyteArray)env->CallStaticObjectMethod(cls, method, jUrl);
jbyteArray byteData = (jbyteArray)env->CallStaticObjectMethod(cls, method, jUrl);

if (env->IsSameObject(byteData, NULL)) {
throw std::runtime_error("Error fetching data from a url");
Expand All @@ -68,13 +63,12 @@ void ETInstallerModule::injectJSIBindings() {

auto _isEmulator = isEmulator();

RnExecutorchInstaller::injectJSIBindings(jsiRuntime_, jsCallInvoker_,
fetchDataByUrl, _isEmulator);
RnExecutorchInstaller::injectJSIBindings(jsiRuntime_, jsCallInvoker_, fetchDataByUrl,
_isEmulator);
}
} // namespace rnexecutorch

JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
rnexecutorch::java_machine = vm;
return facebook::jni::initialize(
vm, [] { rnexecutorch::ETInstallerModule::registerNatives(); });
}
return facebook::jni::initialize(vm, [] { rnexecutorch::ETInstallerModule::registerNatives(); });
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,11 @@ using namespace react;

class ETInstallerModule : public jni::HybridClass<ETInstallerModule> {
public:
static auto constexpr kJavaDescriptor =
"Lcom/swmansion/rnexecutorch/ETInstaller;";
static auto constexpr kJavaDescriptor = "Lcom/swmansion/rnexecutorch/ETInstaller;";

static jni::local_ref<ETInstallerModule::jhybriddata>
initHybrid(jni::alias_ref<jhybridobject> jThis, jlong jsContext,
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>
jsCallInvokerHolder);
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder);

static void registerNatives();

Expand All @@ -33,10 +31,9 @@ class ETInstallerModule : public jni::HybridClass<ETInstallerModule> {
jsi::Runtime *jsiRuntime_;
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;

explicit ETInstallerModule(
jni::alias_ref<ETInstallerModule::jhybridobject> &jThis,
jsi::Runtime *jsiRuntime,
const std::shared_ptr<facebook::react::CallInvoker> &jsCallInvoker);
explicit ETInstallerModule(jni::alias_ref<ETInstallerModule::jhybridobject> &jThis,
jsi::Runtime *jsiRuntime,
const std::shared_ptr<facebook::react::CallInvoker> &jsCallInvoker);
};

} // namespace rnexecutorch
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@ inline bool isEmulator() {
std::string result;
__system_property_read_callback(
pi,
[](void *cookie, const char * /*__name*/, const char *value,
uint32_t /*__serial*/) {
[](void *cookie, const char * /*__name*/, const char *value, uint32_t /*__serial*/) {
*static_cast<std::string *>(cookie) = value;
},
&result);
Expand Down
Loading
Loading