Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 219 additions & 0 deletions QuadTree.Benchmark/AVLSet.fs
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
namespace QuadTree.Benchmarks.AVLSet

open BenchmarkDotNet.Attributes
open BenchmarkDotNet.Configs
open QuadTree.AVLSet
open QuadTree.AVLSet.Parallel

[<GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)>]
Comment thread
gsvgit marked this conversation as resolved.
[<CategoriesColumn>]
[<HtmlExporter>]
[<MemoryDiagnoser>]
type SingleOpsBenchmark() =
let rnd = System.Random(1234561)

[<Params(100, 10000, 100000)>]
[<DefaultValue>]
val mutable public A: int

[<DefaultValue>]
val mutable public rndInt: int

[<DefaultValue>]
val mutable public setA: AVLSet<int>

[<GlobalSetup>]
member self.Setup() =
self.rndInt <- rnd.Next(self.A + 1, self.A + 1000)

let dataA = Array.init self.A (fun _ -> rnd.Next())

self.setA <-
dataA
|> Array.fold
(fun (set: AVLSet<int>) v ->
match AVLSet.add v set with
| Ok nextSet -> nextSet
| Error err -> failwithf "Benchmark setup failed: %A" err)
AVLSet.empty

[<Benchmark>]
[<BenchmarkCategory("Adding")>]
member self.AddingOneElement() = AVLSet.add self.rndInt self.setA

[<Benchmark>]
[<BenchmarkCategory("Deleting")>]
member self.DeletingOneElement() = AVLSet.delete self.rndInt self.setA


[<GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)>]
[<CategoriesColumn>]
[<HtmlExporter>]
[<MemoryDiagnoser>]
type SequentialSetsBenchmark() =
let rnd = System.Random(1234561)

[<Params(100, 10000, 100000)>]
[<DefaultValue>]
val mutable public A: int

[<Params(100, 10000)>]
[<DefaultValue>]
val mutable public B: int

[<DefaultValue>]
val mutable public setA: AVLSet<int>

[<DefaultValue>]
val mutable public setB: AVLSet<int>

[<GlobalSetup>]
member self.Setup() =
let dataA = Array.init self.A (fun _ -> rnd.Next())

let dataB = Array.init self.B (fun _ -> rnd.Next())

self.setA <-
dataA
|> Array.fold
(fun (set: AVLSet<int>) v ->
match AVLSet.add v set with
| Ok nextSet -> nextSet
| Error err -> failwithf "Benchmark setup failed: %A" err)
AVLSet.empty

self.setB <-
dataB
|> Array.fold
(fun (set: AVLSet<int>) v ->
match AVLSet.add v set with
| Ok nextSet -> nextSet
| Error err -> failwithf "Benchmark setup failed: %A" err)
AVLSet.empty

[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Union")>]
member self.SequentialUnion() = AVLSet.union self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Union")>]
member self.UnionViaTreeTraversal() =
AVLSet.Traversal.union self.setA self.setB

[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Intersection")>]
member self.SequentialIntersection() = AVLSet.intersection self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Intersection")>]
member self.IntersectionViaTreeTraversal() =
AVLSet.Traversal.intersection self.setA self.setB

[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Difference")>]
member self.SequentialDifference() = AVLSet.difference self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Difference")>]
member self.DifferenceViaTreeTraversal() =
AVLSet.Traversal.difference self.setA self.setB

[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Symmetrical Difference")>]
member self.SequentialSymmetricalDifference() =
AVLSet.symmDifference self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Symmetrical Difference")>]
member self.SymmetricalDifferenceViaTreeTraversal() =
AVLSet.Traversal.symmDifference self.setA self.setB


[<ShortRunJob>]
[<GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)>]
[<CategoriesColumn>]
[<HtmlExporter>]
[<MemoryDiagnoser>]
[<ThreadingDiagnoser>]
type ParallelSetsBenchmark() =
let rnd = System.Random(1234561)

[<Params(1000, 10000, 100000)>]
[<DefaultValue>]
val mutable public A: int

[<Params(100, 10000)>]
[<DefaultValue>]
val mutable public B: int

[<Params(1, 2, 4)>]
[<DefaultValue>]
val mutable public threads: int

[<DefaultValue>]
val mutable public setA: AVLSet<int>

[<DefaultValue>]
val mutable public setB: AVLSet<int>

[<GlobalSetup>]
member self.Setup() =
let dataA = Array.init self.A (fun _ -> rnd.Next())

let dataB = Array.init self.B (fun _ -> rnd.Next())

self.setA <-
dataA
|> Array.fold
(fun set v ->
match AVLSet.add v set with
| Ok s -> s
| Error e -> failwithf "%A" e)
AVLSet.empty

self.setB <-
dataB
|> Array.fold
(fun set v ->
match AVLSet.add v set with
| Ok s -> s
| Error e -> failwithf "%A" e)
AVLSet.empty


[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Union")>]
member self.SequentialUnion() = AVLSet.union self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Union")>]
member self.ParallelUnionWithThreads() =
ParallelAVLSet.union (Some self.threads) self.setA self.setB

[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Intersection")>]
member self.SequentialIntersection() = AVLSet.intersection self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Intersection")>]
member self.ParallelIntersectionWithThreads() =
ParallelAVLSet.intersection (Some self.threads) self.setA self.setB

[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Difference")>]
member self.SequentialDifference() = AVLSet.difference self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Difference")>]
member self.ParallelDifferenceWithThreads() =
ParallelAVLSet.difference (Some self.threads) self.setA self.setB

[<Benchmark(Baseline = true)>]
[<BenchmarkCategory("Symmetrical Difference")>]
member self.SequentialSymmetricalDifference() =
AVLSet.symmDifference self.setA self.setB

[<Benchmark>]
[<BenchmarkCategory("Symmetrical Difference")>]
member self.ParallelSymmetricalDifferenceWithThreads() =
ParallelAVLSet.symmDifference (Some self.threads) self.setA self.setB
5 changes: 4 additions & 1 deletion QuadTree.Benchmark/Main.fs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ let main argv =
BenchmarkSwitcher
[| typeof<QuadTree.Benchmarks.BFS.Benchmark>
typeof<QuadTree.Benchmarks.SSSP.Benchmark>
typeof<QuadTree.Benchmarks.Triangles.Benchmark> |]
typeof<QuadTree.Benchmarks.Triangles.Benchmark>
typeof<QuadTree.Benchmarks.AVLSet.SingleOpsBenchmark>
typeof<QuadTree.Benchmarks.AVLSet.SequentialSetsBenchmark>
typeof<QuadTree.Benchmarks.AVLSet.ParallelSetsBenchmark> |]

benchmarks.Run argv |> ignore
0
3 changes: 2 additions & 1 deletion QuadTree.Benchmark/QuadTree.Benchmark.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
</PropertyGroup>

<ItemGroup>
<Compile Include="AVLSet.fs"/>
<Compile Include="Utils.fs"/>
<Compile Include="BFS.fs"/>
<Compile Include="SSSP.fs"/>
Expand All @@ -22,4 +23,4 @@
<ProjectReference Include="..\QuadTree\QuadTree.fsproj" />
</ItemGroup>

</Project>
</Project>
71 changes: 70 additions & 1 deletion QuadTree.Benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ Benchmarking infrastructure for
* [BFS](BFS.fs)
* [SSSP](SSSP.fs)
* [Triangles counting](Triangles.fs)
* [AVLSet](AVLSet.fs)

## Steps to run

Expand All @@ -15,4 +16,72 @@ Benchmarking infrastructure for
```
3. Ensure the matrix reader is correctly configured. In ```LoadMatrix ()``` , you can pass a boolean flag to ```readMtx``` indicating whether the matrix should be treated as a directed or undirected graph. Current configuration: undirected for all ```BFS```, ```SSSP``` and ```Triangles counting```.
4. Run evaluation: ```dotnet run -c Release -- --filter '*.SSSP.*'``` You can use ```--filter``` to specify particular benchmarks. Use ```--filter '*'``` to run all available benchmarks.
5. Raw benchmarking results are saved in ```BenchmarkDotNet.Artifacts/results/*.csv```.
5. Raw benchmarking results are saved in ```BenchmarkDotNet.Artifacts/results/*.csv```.

### AVLSet

Benchmarking the `AVLSet` data structure operations.

**Tested operations:**
- `Adding` and `Deleting` single elements.
- Set operations: `Union`, `Intersection`, `Difference`, `Symmetrical Difference`.

For set operations, three implementations are compared:
- **Sequential:** Standard sequential operations (used as the Baseline).
- **Tree Traversal:** Optimized operations using tree traversal.
- **Parallel:** Multi-threaded operations.

**Parameters evaluated:**
- `A`: Size of the primary set (100; 10,000; 1,000,000).
- `B`: Size of the secondary set (100; 1,000; 100,000).
- `DataTypeA`: Data distribution for the primary set (`Random` or `Sorted`).
- `threads`: Number of threads allocated for parallel operations (1, 2, 4, 8).

**How to run AVLSet benchmarks:**
To run only the AVLSet benchmarks, use the following command:
`dotnet run -c Release --filter '*AVLSet*'`

---

### Benchmark Results

Based on the benchmarking data obtained via BenchmarkDotNet, we can draw comprehensive architectural conclusions regarding asymptotic complexity, algorithmic trade-offs, and multi-threading overhead in immutable data structures.

#### 1. Single Element Operations: Asymptotic Complexity Validation
Operations for single elements (`Adding`, `Deleting`) perfectly demonstrate the expected logarithmic **$O(\log N)$** time complexity associated with balanced AVL trees.
* **Execution Time:** Increasing the tree size by a factor of 1,000 (from $100$ to $100,000$ elements) only increases execution time by approximately **2.3x** (e.g., adding an element scales from $582.1$ ns to $1,376.7$ ns).
* **Memory Allocations:** Memory consumption also scales logarithmically. Adding an element to a tree of $100$ nodes allocates $880$ Bytes, while a tree of $100,000$ nodes requires only $2,080$ Bytes. This perfectly reflects the cost of path-copying (creating new nodes from the inserted leaf up to the root) in immutable tree structures.

#### 2. Tree Traversal vs. Sequential (Algorithmic Trade-offs)
The `Traversal` optimization yields highly polarized results depending on the specific operation and the ratio between the sizes of sets $A$ and $B$.

* **The Triumphs (`Intersection`):** Traversal completely dominates standard sequential intersection when set sizes are heavily skewed. For $A=100,000$ and $B=100$, Traversal takes **$83.9$ μs** compared to Sequential's **$318.2$ μs** (a **~3.8x speedup**). It bypasses deep recursive merges and instead maps the smaller set against the larger one efficiently.
* **The Catastrophes (`Difference` & `Symmetrical Difference`):** Traversal causes catastrophic algorithmic degradation for difference operations when applied to the wrong set ratios. For instance, calculating the Difference for $A=100$ and $B=10,000$ takes Sequential operations $168.1$ μs, while Traversal takes **$6,958.6$ μs** (a massive **41.3x slowdown**). This highlights the cost of blindly traversing a large tree to perform sequential lookups.
* **Conclusion:** The `Traversal` strategy should only be conditionally invoked using heuristics.

#### 3. The Parallel Slowdown Phenomenon
The multi-threaded implementation (`ParallelAVLSet`) was benchmarked across 1, 2, and 4 threads. Counter-intuitively, the parallel implementation consistently underperforms the sequential baseline across all metrics (time and memory), providing an example of **Parallel Slowdown**.

* **Task Explosion:** The recursive divide-and-conquer strategy spawns an excessive number of micro-tasks. In the `Intersection` benchmark ($A=100k, B=10k$), the parallel execution generated **$71,315$ completed work items** for a single operation. The overhead of scheduling and synchronizing these micro-tasks in the Thread Pool entirely eclipses the actual computational work.
* **Core Contention:** In almost all scenarios, allocating *more* threads worsened the execution time. For $A=10k, B=10k$ Difference, running on 1 thread took $16.7$ ms, but spreading it across 2 threads spiked the time to **$62.1$ ms**. This indicates severe CPU cache trashing, lock contention, and context switching penalties.
* **GC Thrashing & Memory Pressure:** Parallelizing immutable tree operations causes a severe allocation spike. In extreme cases, parallel operations allocated over **$100$ MB** of memory (compared to $18.4$ MB for Sequential), triggering over **17,000 Gen0 Garbage Collection cycles** in a single benchmark run. The GC "stop-the-world" pauses negate any multi-core benefits.
* **Architectural Takeaway:** Fine-grained parallelism is unsuited for lightweight immutable tree operations. To make parallelization viable in the future, a **Granularity Threshold** must be implemented (e.g., falling back to sequential execution for subtrees with fewer than $5,000$ nodes) to drastically reduce task overhead.

#### Table

| Operation Scenario (A × B) | Implementation Type | Execution Time | Memory Allocated | Ratio | Algorithmic Insight |
| --- | --- | --- | --- | --- | --- |
| **Single Add** (100) | Sequential Baseline | 582.1 ns | 880 B | 1.00 (Base) | Logarithmic $O(\log N)$ baseline. |
| **Single Add** (100,000) | Sequential Baseline | 1,376.7 ns | 2,080 B | ~2.3x scales | Expected path-copying cost. |
| --- | --- | --- | --- | --- | --- |
| **Intersection** (100k × 100) | Sequential Baseline | 318.25 μs | 447.82 KB | 1.00 (Base) | Standard recursive merge. |
| **Intersection** (100k × 100) | Tree Traversal | **83.99 μs** | **86.54 KB** | **~3.8x Speedup** | **Optimal Heuristic:** Huge $A \gg B$ asymmetry. |
| --- | --- | --- | --- | --- | --- |
| **Difference** (100 × 10k) | Sequential Baseline | 168.15 μs | 230.23 KB | 1.00 (Base) | Efficient baseline difference. |
| **Difference** (100 × 10k) | Tree Traversal | 6,958.68 μs | 8.86 MB | **41.39x Slowdown** | **Catastrophic Degradation:** Wrong tree ratio. |
| --- | --- | --- | --- | --- | --- |
| **Difference** (10k × 10k) | Sequential Baseline | 5.68 ms | 6.41 MB | 1.00 (Base) | Balanced trees sequential. |
| **Difference** (10k × 10k) | Parallel (4 Threads) | 57.65 ms | 26.66 MB | **10.28x Slowdown** | High Thread Pool & GC contention. |
| --- | --- | --- | --- | --- | --- |
| **Intersection** (100k × 10k) | Sequential Baseline | 16.29 ms | 18.45 MB | 1.00 (Base) | Large scale baseline. |
| **Intersection** (100k × 10k) | Parallel (4 Threads) | 357.84 ms | 103.17 MB | **22.03x Slowdown** | **Max Task Explosion:** 71,300+ tasks created. |
5 changes: 4 additions & 1 deletion QuadTree.Tests/QuadTree.Tests.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
</PropertyGroup>

<ItemGroup>
<Compile Include="Tests.AVLSet.fs" />
<Compile Include="Tests.fs" />
<Compile Include="Tests.Vector.fs" />
<Compile Include="Tests.Matrix.fs" />
Expand All @@ -18,6 +19,8 @@

<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.4" />
<PackageReference Include="FsUnit.xUnit" Version="6.0.1" />
Comment thread
gsvgit marked this conversation as resolved.
<PackageReference Include="FsCheck.Xunit" Version="3.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
Expand All @@ -27,4 +30,4 @@
<ProjectReference Include="..\QuadTree\QuadTree.fsproj" />
</ItemGroup>

</Project>
</Project>
Loading