From 209ece035a3cacb4a1e18f44f410b5ed8748fce4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:25:49 +0000 Subject: [PATCH 01/33] fix: crashes in documented CLI options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six commands taken verbatim from the docs and --help output segfault. `-e print`, documented in doc/quickstart_cachesim.md, dereferences state that parse_params runs before: SLRU reads lru_max_n_bytes[0] before the default segment sizes are allocated, and QDLP/S3FIFOd read the sub-cache name before the sub-cache is built. Report the effective configuration instead — the even split for SLRU, the configured cache type for the other two. SLRU_current_params also stops appending once the buffer is full rather than passing a negative size to snprintf. `-o PATH` in traceAnalyzer and mrcProfiler is declared OPTION_ARG_OPTIONAL, so argp hands the handler a NULL arg for the space-separated form and strncpy dereferences it; only `-oPATH` and `--output=PATH` worked. The same applied to mrcProfiler's --algo, --size, --profiler and --profiler-params. These all require a value, so drop OPTION_ARG_OPTIONAL and give them argument names in --help. `--verbose` with no value reaches is_true(NULL) and crashes in strcasecmp. Bare flags are the intended usage for OPTION_ARG_OPTIONAL entries, so treat a NULL arg as true. Verified: all 37 runnable commands in the quickstart docs now succeed, and ctest passes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/bin/cli_reader_utils.c | 6 +++++ libCacheSim/bin/mrcProfiler/cli_parser.cpp | 10 ++++---- libCacheSim/bin/traceAnalyzer/cli_parser.cpp | 5 ++-- libCacheSim/cache/eviction/QDLP.c | 6 ++++- libCacheSim/cache/eviction/S3FIFOd.c | 6 ++++- libCacheSim/cache/eviction/SLRU.c | 24 ++++++++++++++++---- 6 files changed, 43 insertions(+), 14 deletions(-) diff --git a/libCacheSim/bin/cli_reader_utils.c b/libCacheSim/bin/cli_reader_utils.c index eff306813..4b36a6fdf 100644 --- a/libCacheSim/bin/cli_reader_utils.c +++ b/libCacheSim/bin/cli_reader_utils.c @@ -58,6 +58,12 @@ trace_type_e trace_type_str_to_enum(const char *trace_type_str, } bool is_true(const char *arg) { + /* options declared OPTION_ARG_OPTIONAL are passed a NULL arg when the bare + * flag is used, e.g. `--verbose`; treat the flag's presence as true */ + if (arg == NULL) { + return true; + } + if (strcasecmp(arg, "true") == 0 || strcasecmp(arg, "1") == 0 || strcasecmp(arg, "yes") == 0 || strcasecmp(arg, "y") == 0) { return true; diff --git a/libCacheSim/bin/mrcProfiler/cli_parser.cpp b/libCacheSim/bin/mrcProfiler/cli_parser.cpp index 43f2788f3..18e810bbb 100644 --- a/libCacheSim/bin/mrcProfiler/cli_parser.cpp +++ b/libCacheSim/bin/mrcProfiler/cli_parser.cpp @@ -73,24 +73,24 @@ static struct argp_option options[] = { 1}, {NULL, 0, NULL, 0, "mrc profiler options:", 0}, - {"algo", OPTION_CACHE_ALGORITHM, "LRU", OPTION_ARG_OPTIONAL, + {"algo", OPTION_CACHE_ALGORITHM, "ALGO", 0, "Which algorithm to profile. Only Support LRU for SHARDS.", 2}, - {"size", OPTION_MRC_SIZE, "0.01,1,100", OPTION_ARG_OPTIONAL, + {"size", OPTION_MRC_SIZE, "SIZES", 0, "MRC profile size. Support two formats " "[start_size,end_size,#test_points|size1,size2,size3,...,size_n]. For " "size settings, both explicit sizes (e.g., 1GiB) and WSS-based sizes (a " "floating-point number between 0 and 1) are supported.", 2}, - {"profiler", OPTION_PROFILER, "SHARDS", OPTION_ARG_OPTIONAL, + {"profiler", OPTION_PROFILER, "PROFILER", 0, "Which profiler to use. Support SHARDS|MINISIM", 2}, - {"profiler-params", OPTION_PROFILER_PARAMS, "", OPTION_ARG_OPTIONAL, + {"profiler-params", OPTION_PROFILER_PARAMS, "PARAMS", 0, "Profiler parameters. ", 2}, {"ignore-obj-size", OPTION_IGNORE_OBJ_SIZE, NULL, OPTION_ARG_OPTIONAL, "Ignore object size", 2}, {NULL, 0, NULL, 0, "common parameters:", 0}, - {"output", OPTION_OUTPUT_PATH, "", OPTION_ARG_OPTIONAL, "Output path", 3}, + {"output", OPTION_OUTPUT_PATH, "PATH", 0, "Output path", 3}, {"verbose", OPTION_VERBOSE, NULL, OPTION_ARG_OPTIONAL, "Produce verbose output", 3}, {NULL, 0, NULL, 0, NULL, 0}}; diff --git a/libCacheSim/bin/traceAnalyzer/cli_parser.cpp b/libCacheSim/bin/traceAnalyzer/cli_parser.cpp index 588e48665..7251abc92 100644 --- a/libCacheSim/bin/traceAnalyzer/cli_parser.cpp +++ b/libCacheSim/bin/traceAnalyzer/cli_parser.cpp @@ -110,7 +110,7 @@ static struct argp_option options[] = { {NULL, 0, NULL, 0, "common parameters:", 0}, - {"output", OPTION_OUTPUT_PATH, "", OPTION_ARG_OPTIONAL, "Output path", 8}, + {"output", OPTION_OUTPUT_PATH, "PATH", 0, "Output path", 8}, {"verbose", OPTION_VERBOSE, NULL, OPTION_ARG_OPTIONAL, "Produce verbose output", 8}, {NULL, 0, NULL, 0, NULL, 0}}; @@ -219,7 +219,8 @@ static char args_doc[] = "trace_path trace_type [--task1] [--task2] ..."; /* Program documentation. */ static char doc[] = - "example: ./bin/traceAnalyzer ../data/trace.vscsi vscsi --common\n\n" + "example: ./bin/traceAnalyzer ../data/cloudPhysicsIO.vscsi vscsi " + "--common\n\n" "trace_type: txt/csv/twr/vscsi/oracleGeneralBin and more\n" "if using csv trace, considering specifying -t obj-id-is-num=true\n\n" "task: " diff --git a/libCacheSim/cache/eviction/QDLP.c b/libCacheSim/cache/eviction/QDLP.c index b1edd82fd..a1eb19890 100644 --- a/libCacheSim/cache/eviction/QDLP.c +++ b/libCacheSim/cache/eviction/QDLP.c @@ -437,8 +437,12 @@ static inline bool QDLP_can_insert(cache_t *cache, const request_t *req) { // *********************************************************************** static const char *QDLP_current_params(QDLP_params_t *params) { static __thread char params_str[128]; + /* main_cache is only built after the parameters are parsed, so report the + * configured type, which is what `-e print` runs against */ snprintf(params_str, 128, "fifo-size-ratio=%.4lf,main-cache=%s\n", - params->small_size_ratio, params->main_cache->cache_name); + params->small_size_ratio, + params->main_cache == NULL ? params->main_cache_type + : params->main_cache->cache_name); return params_str; } diff --git a/libCacheSim/cache/eviction/S3FIFOd.c b/libCacheSim/cache/eviction/S3FIFOd.c index f04b8c75c..79826721d 100644 --- a/libCacheSim/cache/eviction/S3FIFOd.c +++ b/libCacheSim/cache/eviction/S3FIFOd.c @@ -530,8 +530,12 @@ static inline bool S3FIFOd_can_insert(cache_t *cache, const request_t *req) { // *********************************************************************** static const char *S3FIFOd_current_params(S3FIFOd_params_t *params) { static __thread char params_str[128]; + /* main_fifo is only built after the parameters are parsed, so report the + * configured type, which is what `-e print` runs against */ snprintf(params_str, 128, "fifo-size-ratio=%.4lf,main-cache=%s\n", - params->small_fifo_size_ratio, params->main_fifo->cache_name); + params->small_fifo_size_ratio, + params->main_fifo == NULL ? params->main_fifo_type + : params->main_fifo->cache_name); return params_str; } diff --git a/libCacheSim/cache/eviction/SLRU.c b/libCacheSim/cache/eviction/SLRU.c index 75c1136df..8d0d9a67a 100644 --- a/libCacheSim/cache/eviction/SLRU.c +++ b/libCacheSim/cache/eviction/SLRU.c @@ -403,14 +403,28 @@ static bool SLRU_remove(cache_t *cache, obj_id_t obj_id) { // **** parameter set up functions **** // **** **** // *********************************************************************** +/* share of the cache given to one segment, in percent. lru_max_n_bytes is only + * allocated after the parameters are parsed, so it is still NULL when the user + * asks for the parameters with `-e print`; until seg-size says otherwise the + * segments are evenly sized */ +static int SLRU_seg_pct(const cache_t *cache, const SLRU_params_t *params, + const int seg) { + if (params->lru_max_n_bytes == NULL) { + return 100 / params->n_seg; + } + return (int)(params->lru_max_n_bytes[seg] * 100 / cache->cache_size); +} + static const char *SLRU_current_params(cache_t *cache, SLRU_params_t *params) { static __thread char params_str[128]; - int n = snprintf(params_str, 128, "n-seg=%d,seg-size=%d", params->n_seg, - (int)(params->lru_max_n_bytes[0] * 100 / cache->cache_size)); - for (int i = 1; i < params->n_seg; i++) { - n += snprintf(params_str + n, 128 - n, ":%d", - (int)(params->lru_max_n_bytes[i] * 100 / cache->cache_size)); + int n = snprintf(params_str, sizeof(params_str), "n-seg=%d,seg-size=%d", + params->n_seg, SLRU_seg_pct(cache, params, 0)); + + for (int i = 1; i < params->n_seg && n > 0 && n < (int)sizeof(params_str); + i++) { + n += snprintf(params_str + n, sizeof(params_str) - n, ":%d", + SLRU_seg_pct(cache, params, i)); } return params_str; From e5929d04599dab588f71ba4f661dbb54f4bafe0d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:26:04 +0000 Subject: [PATCH 02/33] docs: point examples at trace files that exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docs referenced data/trace.vscsi, data/trace.csv, data/trace.txt and data/trace.oracleGeneral throughout. None of those exist — the sample traces are data/cloudPhysicsIO.*. Every copy-pasted quickstart command failed on a fresh clone. The README was fixed in #321; this does the same for doc/. Also in this pass: - run the examples as ./bin/ from the build directory, matching the README, instead of ./ with a ../data path that only resolves one directory up from the binary - add obj-id-is-num=true to the csv examples; the sample trace has a numeric id column and cachesim errors out without it - fix links to GDSF, LHD_Interface.cpp, the eviction CMakeLists and the example trace reader, none of which resolved - note that GLCache/LRB/3LCache need their build flags, so "do not support algorithm" is explainable - rewrite FAQ.md: correct the struct field name (clock_time, not real_time), fix the dead data/trace.csv link, and add entries for the optional-build error and where to get real traces Every command in the quickstart docs was run against the sample traces. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- .github/copilot-instructions.md | 2 +- FAQ.md | 48 ++++++++++++++---- doc/README.md | 6 +++ doc/advanced_lib.md | 4 +- doc/advanced_lib_extend.md | 4 +- doc/quickstart_cachesim.md | 86 ++++++++++++++++++--------------- doc/quickstart_traceAnalyzer.md | 6 +-- doc/quickstart_traceUtils.md | 2 +- 8 files changed, 100 insertions(+), 58 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ed9c8e72f..d723f6712 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -17,7 +17,7 @@ - Use a standard out-of-source CMake build for release-style work: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release` then `cmake --build build`. - CMake tests are enabled by default. Run `ctest --test-dir _build_dbg --output-on-failure` after the debug build, or the equivalent `build/` test directory if you used a separate build tree. - If you change installation, packaging, or the public library surface, also review `test/test_lib.sh`. -- Use sample traces in `data/` for quick validation unless the task specifically requires the large traces in `2024_google/`. +- Use sample traces in `data/` for quick validation. They are deliberately tiny, so never use them to compare miss ratios between algorithms; larger traces are listed at https://github.com/cacheMon/cache_dataset. ## Project-Specific Conventions - When adding a new eviction algorithm, reader, or plugin, follow `doc/advanced_lib_extend.md` instead of inventing a new integration path. These changes usually require updates to implementation files, registration headers, CMake lists, CLI/parser wiring, and tests. diff --git a/FAQ.md b/FAQ.md index 1ff4995b7..b50461f95 100644 --- a/FAQ.md +++ b/FAQ.md @@ -1,17 +1,47 @@ -## FAQ -1. **how to read OracleGeneral trace，how to transform from csv to it? ** -The [oracleGeneral](/libCacheSim/traceReader/customizedReader/oracle/oracleGeneralBin.h) trace is a binary trace, so you cannot direct read as txt file. Each request uses the following data struct +# FAQ + +### How do I read an oracleGeneral trace, and how do I convert a csv trace into one? + +The [oracleGeneral](/libCacheSim/traceReader/customizedReader/oracle/oracleGeneralBin.h) trace is a binary format, so it cannot be read as a text file. Each request is the following struct: + ```c struct { - uint32_t real_time; + uint32_t clock_time; uint64_t obj_id; uint32_t obj_size; - int64_t next_access_vtime; + int64_t next_access_vtime; // -1 if there is no next access }; ``` -* Read the trace: we have provided a tool `tracePrint` that you can use to print the trace in plain text, it is compiled and under `bin/` -* Convert csv to oracleGeneral: we have provided `traceConv` to convert traces. The help menu should be sufficient to get started. +* **Read the trace**: use `tracePrint` to print the trace as plain text. It is built into `bin/` alongside `cachesim`. + ```bash + ./bin/tracePrint ../data/cloudPhysicsIO.oracleGeneral.bin oracleGeneral + ``` +* **Convert a csv trace**: use `traceConv`. See [quickstart_traceUtils.md](/doc/quickstart_traceUtils.md), or run `./bin/traceConv --help`. + ```bash + ./bin/traceConv ../data/cloudPhysicsIO.csv csv \ + -t "time-col=2,obj-id-col=5,obj-size-col=4,obj-id-is-num=1" \ + --output-format=oracleGeneral + ``` + +oracleGeneral traces are usually stored zstd-compressed, and libCacheSim reads them without decompressing first. + +### What are the units in a trace? + +In the sample [cloudPhysicsIO.csv](/data/cloudPhysicsIO.csv), time is in seconds and object size is in bytes. + +`next_access_vtime` is a *logical* time: the number of requests between the current request and the next request to the same object, or `-1` when the object is never accessed again. Algorithms that need future information, such as [Belady](/libCacheSim/cache/eviction/Belady.c) and BeladySize, rely on it, which is why they only work on oracle traces. + +Object ids are hashed unless the reader is told they are already numeric. Pass `obj-id-is-num=true` in `--trace-type-params` when the id column holds numbers — `cachesim` stops with an error if you leave it out on such a trace. + +### Why does `cachesim` say "do not support algorithm X"? + +Some algorithms are behind an optional build flag because they pull in extra dependencies: GLCache (`-DENABLE_GLCACHE=ON`), LRB (`-DENABLE_LRB=ON`), and 3LCache (`-DENABLE_3L_CACHE=ON`). Rebuild with the relevant flag to enable them. See the [README](/README.md#supported-algorithms) for the full list. + +### Where can I get larger traces? + +The traces in [data/](/data/) are samples and are **far too small to compare miss ratios between algorithms**. We maintain a list of open-source cache datasets at [cacheMon/cache_dataset](https://github.com/cacheMon/cache_dataset). + +--- -2. **What are the units in the trace? ** -In the [trace.csv](/data/trace.csv), the time unit is in sec, the next_access_time is the logical time (# requests) between current and the next request (to the same object). The next access time is used by some algorithms that require future information, e.g., Belady. The object id is a hash of raw object id (string or numeric value). +More questions? Check the [documentation index](/doc/README.md), search the [issue tracker](https://github.com/1a1a11a/libCacheSim/issues), or ask in [Discussions](https://github.com/1a1a11a/libCacheSim/discussions). diff --git a/doc/README.md b/doc/README.md index b95f80720..9c6cc53f8 100644 --- a/doc/README.md +++ b/doc/README.md @@ -18,3 +18,9 @@ ## Developer Documentation - [Debugging Guide](debug.md) - [Install & Build](install.md) +- [Contributing](/CONTRIBUTING.md) + +## Help +- [FAQ](/FAQ.md) +- [Issue tracker](https://github.com/1a1a11a/libCacheSim/issues) +- [Discussions](https://github.com/1a1a11a/libCacheSim/discussions) diff --git a/doc/advanced_lib.md b/doc/advanced_lib.md index 335f9da29..1e0db26b0 100644 --- a/doc/advanced_lib.md +++ b/doc/advanced_lib.md @@ -116,13 +116,13 @@ open_trace(data_path, PLAIN_TXT_TRACE, NULL); ```c reader_init_param_t init_params_csv = {.delimiter=',', .time_field=2, .obj_id_field=6, .obj_size_field=4, .has_header=FALSE}; -reader_t *reader_csv_c = open_trace("data/trace.csv", CSV_TRACE, &init_params_csv); +reader_t *reader_csv_c = open_trace("data/cloudPhysicsIO.csv", CSV_TRACE, &init_params_csv); ``` ##### Setup a binary reader ```c reader_init_param_t init_params_bin = {.binary_fmt="<3I2H2Q", .obj_size_field=2, .obj_id_field=6, }; -reader_t *reader_bin_l = setup_reader("data/trace.vscsi", BIN_TRACE, &init_params_bin); +reader_t *reader_bin_l = setup_reader("data/cloudPhysicsIO.vscsi", BIN_TRACE, &init_params_bin); ``` The format of a binary trace is the same as [Python struct format specifier](https://docs.python.org/3/library/struct.html). diff --git a/doc/advanced_lib_extend.md b/doc/advanced_lib_extend.md index d91ac92c2..547751671 100644 --- a/doc/advanced_lib_extend.md +++ b/doc/advanced_lib_extend.md @@ -35,7 +35,7 @@ Specifically, you can following the steps: 2. If your cache eviction algorithm needs extra metadata, add a new object metadata struct in [include/libCacheSim/cacheObj.h](/libCacheSim/include/libCacheSim/cacheObj.h). 3. Add `myCache_init()` function to [include/libCacheSim/evictionAlgo.h](/libCacheSim/include/libCacheSim/evictionAlgo.h). -4. Add mycache.c to [CMakeLists.txt](/libCacheSim/cache/eviction/CMakeLists.txt) so that it can be compiled. +4. Add mycache.c to [CMakeLists.txt](/libCacheSim/cache/CMakeLists.txt) so that it can be compiled. 5. Add command line option in [bin/cachesim/cache_init.h](/libCacheSim/bin/cachesim/cache_init.h) so that you can use `cachesim` binary. You may also want to take a look at [bin/cachesim/cli_parser.c](/libCacheSim/bin/cachesim/cli_parser.c). 6. Remember to add a test in [test/test_evictionAlgo.c](/test/test_evictionAlgo.c) and add the algorithm to this [README](README.md). @@ -66,7 +66,7 @@ There are two steps you can follow, libCacheSim supports [txt](/libCacheSim/traceReader/generalReader/txt.c), [csv](/libCacheSim/traceReader/generalReader/csv.c), and binary traces. We prefer binary traces because it allows libCacheSim to run faster, and the traces are more compact. For binary traces, libCacheSim also supports zstd compressed traces without decompression. -But if you ever need to implement a new trace type, please see [here](/libCacheSim/traceReader/customizedReader/akamaiBin.h) for an example reader. +But if you ever need to implement a new trace type, see [twrBin.h](/libCacheSim/traceReader/customizedReader/twrBin.h) for a compact example reader, or [vscsi.h](/libCacheSim/traceReader/customizedReader/vscsi.h) and [oracleGeneralBin.h](/libCacheSim/traceReader/customizedReader/oracle/oracleGeneralBin.h) for the formats used by the sample traces in [data/](/data/). To implement a reader, you need to implement two functions: ```c diff --git a/doc/quickstart_cachesim.md b/doc/quickstart_cachesim.md index 5fee2a253..c4c5bd58e 100644 --- a/doc/quickstart_cachesim.md +++ b/doc/quickstart_cachesim.md @@ -11,16 +11,18 @@ Meanwhile, cachesim has high-performance with low resource usages. --- ## Installation -First, [build libCacheSim](/doc/install.md). After building libCacheSim, `cachesim` should be in the build directory. +First, [build libCacheSim](/doc/install.md). After building libCacheSim, `cachesim` is in the `bin/` subdirectory of your build directory. + +All commands on this page are run from the build directory (`_build/` if you followed the [README](/README.md)), so the sample traces in [data/](/data/) are at `../data/`. --- ## Basic Usage ``` -./cachesim trace_path trace_type eviction_algo cache_size [OPTION...] +./bin/cachesim trace_path trace_type eviction_algo cache_size [OPTION...] ``` -use `./cachesim --help` to get more information. +use `./bin/cachesim --help` to get more information. ### Run a single cache simulation @@ -29,28 +31,28 @@ Note that vscsi is a trace format, we also support csv traces. ```bash # Note that no space between the cache size and the unit, unit is not case sensitive -./cachesim ../data/trace.vscsi vscsi lru 1gb +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 1gb ``` ### Run multiple cache simulations ```bash # Note that there is no space between the cache sizes -./cachesim ../data/trace.vscsi vscsi lru 1mb,16mb,256mb,8gb +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 1mb,16mb,256mb,8gb # Or you can quote the cache sizes -./cachesim ../data/trace.vscsi vscsi lru "1mb, 16mb, 256mb, 8gb" +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru "1mb, 16mb, 256mb, 8gb" # besides absolute cache size, you can also use fraction of working set size -./cachesim ../data/trace.vscsi vscsi lru 0.001,0.01,0.1,0.2 +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 0.001,0.01,0.1,0.2 # besides using byte as the unit, you can also treat all objects having the same size, and the size is the number of objects -./cachesim ../data/trace.vscsi vscsi lru 1000,16000 --ignore-obj-size 1 +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 1000,16000 --ignore-obj-size 1 # new feature: you can run a few algorithms in parallel by concatenating the algorithms -./cachesim ../data/trace.vscsi vscsi fifo,lru,arc,qdlp 0.01 --ignore-obj-size 1 +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi fifo,lru,arc,qdlp 0.01 --ignore-obj-size 1 # run 4*4 simulations in parallel (no more than n_thread at the same time) -./cachesim ../data/trace.vscsi vscsi fifo,lru,arc,qdlp 0.01,0.05,0.1,0.2 --ignore-obj-size 1 +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi fifo,lru,arc,qdlp 0.01,0.05,0.1,0.2 --ignore-obj-size 1 ``` @@ -59,7 +61,7 @@ cachesim can detect the working set of the trace and automatically generate cach You can enable this feature by setting cache size to 0 or auto. ```bash -./cachesim ../data/trace.vscsi vscsi lru auto +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru auto ``` ### Use different eviction algorithms @@ -70,27 +72,30 @@ cachesim supports the following algorithms: * [LFU](/libCacheSim/cache/eviction/LFU.c) * [ARC](/libCacheSim/cache/eviction/ARC.c) * [SLRU](/libCacheSim/cache/eviction/SLRU.c) -* [GDSF](/libCacheSim/cache/eviction/GDSF.c) +* [GDSF](/libCacheSim/cache/eviction/cpp/GDSF.cpp) * [WTinyLFU](/libCacheSim/cache/eviction/WTinyLFU.c) * [LeCaR](/libCacheSim/cache/eviction/LeCaR.c) * [Cacheus](/libCacheSim/cache/eviction/Cacheus.c) * [Hyperbolic](/libCacheSim/cache/eviction/Hyperbolic.c) -* [LHD](/libCacheSim/cache/eviction/LHD/LHDInterface.cpp) -* [GLCache](/libCacheSim/cache/eviction/GLCache/GLCache.c) +* [LHD](/libCacheSim/cache/eviction/LHD/LHD_Interface.cpp) * [Belady](/libCacheSim/cache/eviction/Belady.c) * [BeladySize](/libCacheSim/cache/eviction/BeladySize.c) * [QD-LP](/libCacheSim/cache/eviction/QDLP.c) +* [S3-FIFO](/libCacheSim/cache/eviction/S3FIFO.c), [Sieve](/libCacheSim/cache/eviction/Sieve.c) +* [GLCache](/libCacheSim/cache/eviction/GLCache/GLCache.c) — build with `-DENABLE_GLCACHE=ON` + +See the [README](/README.md#supported-algorithms) for the full list, including the algorithms that are behind an optional build flag (GLCache, LRB, 3LCache). Asking for one that was not compiled in fails with `do not support algorithm `. You can just use the algorithm name as the eviction algorithm parameter, for example ```bash -./cachesim ../data/trace.vscsi vscsi lecar auto -./cachesim ../data/trace.vscsi vscsi hyperbolic auto -./cachesim ../data/trace.vscsi vscsi lhd auto -./cachesim ../data/trace.vscsi vscsi glcache auto +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lecar auto +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi hyperbolic auto +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lhd auto +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi s3fifo auto # belady and beladySize require oracle trace -./cachesim ../data/trace.oracleGeneral oracleGeneral beladySize auto +./bin/cachesim ../data/cloudPhysicsIO.oracleGeneral.bin oracleGeneral beladySize auto ``` @@ -103,28 +108,29 @@ Besides the column information, a csv reader also requires the delimiter and whe cachesim builds in a simple delimiter and header detector, if the detected result is not correct, you can provide the correct information using `delimiter=,`, `has-header=true`. +Object ids are hashed unless you tell the reader they are already numeric, so add `obj-id-is-num=true` when the id column holds numbers — `cachesim` stops with an error if you leave it out on such a trace. The sample `cloudPhysicsIO.csv` has a numeric id column, so every example below sets it. + ```bash # note that the parameters are separated by comma and quoted -./cachesim ../data/trace.csv csv lru 1gb -t "time-col=2, obj-id-col=5, obj-size-col=4" - -# if object id is numeric, then we can pass obj-id-is-num=true to speed up -./cachesim ../data/trace.csv csv lru 1gb -t "time-col=2, obj-id-col=5, obj-size-col=4, obj-id-is-num=true" +./bin/cachesim ../data/cloudPhysicsIO.csv csv lru 1gb -t "time-col=2, obj-id-col=5, obj-size-col=4, obj-id-is-num=true" +# omitting obj-id-is-num on a numeric id column fails with +# [ERROR] csv.c: detect obj_id is numeric, please specify -t 'obj-id-is-num=1' # note that csv trace does not support UTF-8 encoding, only ASCII encoding is supported -./cachesim ../data/trace.csv csv lru 1gb -t "time-col=2, obj-id-col=5, obj-size-col=4, delimiter=,, has-header=true" +./bin/cachesim ../data/cloudPhysicsIO.csv csv lru 1gb -t "time-col=2, obj-id-col=5, obj-size-col=4, obj-id-is-num=true, delimiter=,, has-header=true" ``` Besides csv trace, we also support txt trace and binary trace. ```bash # txt trace is a simple format that stores obj-id in each line -./cachesim ../data/trace.txt txt lru 1gb +./bin/cachesim ../data/cloudPhysicsIO.txt txt lru 1gb # binary trace, format is specified using format string similar to Python struct -./cachesim ../data/trace.vscsi binary lru 1gb -t "format= Date: Thu, 13 Aug 2026 00:26:16 +0000 Subject: [PATCH 03/33] docs: add contributor and community health files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo had issue templates but no CONTRIBUTING.md, code of conduct, or pull request template, and SECURITY.md was the GitHub boilerplate with its instructional comments still in it and no reporting channel. - CONTRIBUTING.md: build, debug and test workflow, the pre-commit hook, code style, where to add a new algorithm or reader, and PR expectations. Branch guidance points at develop, the default branch. - CODE_OF_CONDUCT.md: Contributor Covenant 2.1, enforcement contact via the maintainers in .github/CODEOWNERS. - SECURITY.md: rewritten around GitHub private vulnerability reporting, with what to include and what is in scope — trace parsing is the untrusted input, vendored third-party code is not. - .github/PULL_REQUEST_TEMPLATE.md: change type, how it was tested, and before/after numbers for anything touching miss ratio or throughput. - CITATION.cff: powers GitHub's "Cite this repository" button; S3-FIFO as the preferred citation, plus the OSDI, HotOS and SIEVE papers. README links to all of them from the contributions section. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- .github/PULL_REQUEST_TEMPLATE.md | 36 +++++++++ CITATION.cff | 121 ++++++++++++++++++++++++++++ CODE_OF_CONDUCT.md | 133 +++++++++++++++++++++++++++++++ CONTRIBUTING.md | 92 +++++++++++++++++++++ README.md | 9 ++- SECURITY.md | 36 ++++++--- 6 files changed, 415 insertions(+), 12 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CITATION.cff create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..8f19c47e7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,36 @@ + + +## What does this PR do? + + + +## Type of change + +- [ ] Bug fix +- [ ] New eviction / admission / prefetch algorithm +- [ ] New trace reader or trace format +- [ ] Performance improvement +- [ ] Documentation +- [ ] Build / CI +- [ ] Other: + +## How was it tested? + + + +- [ ] `ctest --test-dir _build --output-on-failure` passes +- [ ] Build is warning-free (CI uses `-Wall -Wextra -Werror`) + +## Results + + + +## Checklist + +- [ ] Formatted with `clang-format` (or the pre-commit hook from `scripts/setup_hooks.sh`) +- [ ] Added or updated tests +- [ ] Added or updated documentation +- [ ] New algorithms are registered in the CLI and listed in the [README](../README.md#supported-algorithms) diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..7674c21d7 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,121 @@ +cff-version: 1.2.0 +title: libCacheSim +message: >- + If you use libCacheSim in your research, please cite the + software and the relevant papers below. +type: software +authors: + - family-names: Yang + given-names: Juncheng + affiliation: Harvard University +repository-code: 'https://github.com/1a1a11a/libCacheSim' +abstract: >- + A high-performance library and set of tools for building + and running cache simulations, analyzing cache traces, and + profiling miss ratio curves. +keywords: + - cache + - caching + - cache simulation + - eviction algorithm + - miss ratio curve + - trace analysis +license: GPL-3.0 +preferred-citation: + type: conference-paper + title: FIFO Queues Are All You Need for Cache Eviction + authors: + - family-names: Yang + given-names: Juncheng + - family-names: Zhang + given-names: Yazhuo + - family-names: Qiu + given-names: Ziyue + - family-names: Yue + given-names: Yao + - family-names: Rashmi + given-names: K. V. + collection-title: >- + Proceedings of the 29th Symposium on Operating Systems + Principles (SOSP '23) + publisher: + name: Association for Computing Machinery + year: 2023 + start: 130 + end: 149 + isbn: '9798400702297' + doi: 10.1145/3600006.3613147 +references: + - type: conference-paper + title: >- + A large-scale analysis of hundreds of in-memory cache + clusters at Twitter + authors: + - family-names: Yang + given-names: Juncheng + - family-names: Yue + given-names: Yao + - family-names: Rashmi + given-names: K. V. + collection-title: >- + 14th USENIX Symposium on Operating Systems Design and + Implementation (OSDI 20) + publisher: + name: USENIX Association + year: 2020 + month: 11 + start: 191 + end: 208 + isbn: '978-1-939133-19-9' + url: 'https://www.usenix.org/conference/osdi20/presentation/yang' + - type: conference-paper + title: >- + FIFO Can Be Better than LRU: The Power of Lazy + Promotion and Quick Demotion + authors: + - family-names: Yang + given-names: Juncheng + - family-names: Qiu + given-names: Ziyue + - family-names: Zhang + given-names: Yazhuo + - family-names: Yue + given-names: Yao + - family-names: Rashmi + given-names: K. V. + collection-title: >- + Proceedings of the 19th Workshop on Hot Topics in + Operating Systems (HotOS '23) + publisher: + name: Association for Computing Machinery + year: 2023 + start: 70 + end: 79 + isbn: '9798400701955' + doi: 10.1145/3593856.3595887 + - type: conference-paper + title: >- + SIEVE is Simpler than LRU: an Efficient Turn-Key + Eviction Algorithm for Web Caches + authors: + - family-names: Zhang + given-names: Yazhuo + - family-names: Yang + given-names: Juncheng + - family-names: Yue + given-names: Yao + - family-names: Vigfusson + given-names: Ymir + - family-names: Rashmi + given-names: K. V. + collection-title: >- + 21st USENIX Symposium on Networked Systems Design and + Implementation (NSDI 24) + publisher: + name: USENIX Association + year: 2024 + month: 4 + start: 1229 + end: 1246 + isbn: '978-1-939133-39-7' + url: 'https://www.usenix.org/conference/nsdi24/presentation/zhang-yazhuo' diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..02b4785f8 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,133 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the maintainers, who are listed in +[.github/CODEOWNERS](.github/CODEOWNERS). Contact them privately — their contact +details are on their GitHub profiles. + +All complaints will be reviewed and investigated promptly and fairly. All +community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..e4db3c3bc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,92 @@ +# Contributing to libCacheSim + +Thanks for your interest in libCacheSim! Bug reports, new algorithms, trace readers, documentation fixes, and performance work are all welcome. + +## Before you start + +* **Small fixes** — typos, broken links, an obvious bug — just open a pull request. +* **Larger changes** — a new algorithm, an API change, a new subsystem — please [open an issue](https://github.com/1a1a11a/libCacheSim/issues) first so we can agree on the approach before you invest the time. +* **Questions** — use [Discussions](https://github.com/1a1a11a/libCacheSim/discussions) rather than the issue tracker. + +## Development setup + +Install the dependencies ([glib](https://developer.gnome.org/glib/), [tcmalloc](https://github.com/google/tcmalloc), [zstd](https://github.com/facebook/zstd)) and build out-of-source: + +```bash +bash scripts/install_dependency.sh # see doc/install.md if this does not work +cmake -G Ninja -B _build -DCMAKE_BUILD_TYPE=Release +ninja -C _build +``` + +Binaries land in `_build/bin/`. The commands in the docs are written to be run from `_build/`, so the sample traces in [`data/`](data/) are at `../data/`: + +```bash +cd _build && ./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 1gb +``` + +For day-to-day work on the C/C++ code, prefer the debug build — it uses the same strict warning flags as CI and skips tcmalloc so it stays debugger-friendly: + +```bash +bash scripts/debug.sh -c # configure + build _build_dbg/ +bash scripts/debug.sh -- data/cloudPhysicsIO.vscsi vscsi lru,s3fifo 100mb,1gb +``` + +See [doc/debug.md](doc/debug.md) for the full debugging workflow. + +### Install the pre-commit hook + +```bash +bash scripts/setup_hooks.sh +``` + +The hook runs clang-format, clang-tidy, and a strict-warning compile on the files you staged, which is roughly what CI will check. Use `SKIP_LINT=1 git commit ...` to bypass it in a pinch. Logs are kept in `.lint-logs/`. + +## Testing + +Tests are CTest-backed and run by default: + +```bash +ctest --test-dir _build --output-on-failure --parallel 4 +``` + +CI additionally builds Ubuntu with LeakSanitizer, so please check that new allocations are freed. If you touch installation, packaging, or the public library surface, also run [`test/test_lib.sh`](test/test_lib.sh). + +Every new eviction, admission, or prefetching algorithm needs a test in the matching file under [`test/`](test/) — for eviction algorithms that is [`test/test_evictionAlgo.c`](test/test_evictionAlgo.c). + +## Code style + +* The project follows **Google style**: 2-space indent, 80-column limit, configured in [`.clang-format`](.clang-format) and [`.clang-tidy`](.clang-tidy). Run `clang-format -i ` before committing. +* Most of the codebase is **C**. Use C++17 only where the surrounding code already does (`cache/eviction/cpp/`, `LHD/`, `LRB/`, the analyzer and profiler binaries). +* **Keep the build warning-free.** CI compiles with `-Wall -Wextra -Werror` plus an extended warning set. +* Cache algorithms, trace readers, and profilers are hot paths — prefer changes that do not add per-request work. + +## Adding something new + +Follow the existing integration path rather than inventing one; these changes usually touch the implementation, a registration header, the CMake lists, the CLI wiring, and a test. + +| What | Guide | +| --- | --- | +| Eviction / admission / prefetch algorithm | [doc/advanced_lib_extend.md](doc/advanced_lib_extend.md) | +| Trace reader | [doc/advanced_lib_extend.md](doc/advanced_lib_extend.md) | +| Algorithm without recompiling (Python/C plugin) | [doc/quickstart_plugin.md](doc/quickstart_plugin.md) | +| Public API change | [doc/API.md](doc/API.md), [doc/advanced_lib.md](doc/advanced_lib.md) | + +When you add an algorithm, also list it in the [README](README.md#supported-algorithms) with the name users pass on the command line. + +## Pull requests + +1. Branch off `develop` — that is the default branch and where PRs are merged. +2. Keep the PR focused; unrelated cleanups are easier to review separately. +3. Make sure `ctest` passes and the build is warning-free. +4. Describe *what* changed and *why*. If it affects miss ratios or throughput, include the numbers and the command you ran. +5. CI must be green before merge. + +## Reporting bugs + +Please use the [bug report template](.github/ISSUE_TEMPLATE/bug_report.md) and include the trace format, the exact command line, and the build type. A reproducer against one of the sample traces in `data/` is ideal, since those ship with the repo. + +Security issues should **not** be filed as public issues — see [SECURITY.md](SECURITY.md). + +## License + +libCacheSim is [GPL-3.0](LICENSE) licensed. By contributing, you agree that your contributions are licensed under the same terms. diff --git a/README.md b/README.md index 029a2ab93..cdeb2de5d 100644 --- a/README.md +++ b/README.md @@ -419,10 +419,13 @@ We provide more comprehensive cache datasets at [https://github.com/cacheMon/cac --- ## Contributions -We gladly welcome pull requests. +We gladly welcome pull requests. See [CONTRIBUTING.md](/CONTRIBUTING.md) for how to build, test, and submit changes, and [doc/advanced_lib_extend.md](/doc/advanced_lib_extend.md) for adding a new algorithm or trace reader. + Before making any large changes, we recommend opening an issue and discussing your proposed changes. If the changes are minor, then feel free to make them without discussion. -This project adheres to Google's coding style. By participating, you are expected to uphold this code. +This project adheres to Google's coding style, and participants are expected to follow our [Code of Conduct](/CODE_OF_CONDUCT.md). + +Found a security issue? Please report it privately — see [SECURITY.md](/SECURITY.md). --- ## Reference @@ -468,6 +471,8 @@ If you used libCacheSim in your research, please cite the above papers. +GitHub's **Cite this repository** button uses [CITATION.cff](/CITATION.cff); [references.md](/references.md) has the same entries as BibTeX, including the SIEVE paper. + --- diff --git a/SECURITY.md b/SECURITY.md index b4e72ae3c..d9562c18f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,19 +2,35 @@ ## Supported Versions - -Security updates are applied only to the most recent release. +Security fixes are applied to the most recent release only. If you are running an older version, please upgrade before reporting. ## Reporting a Vulnerability - +Report privately through GitHub: go to the [Security tab](https://github.com/1a1a11a/libCacheSim/security) of this repository and choose **Report a vulnerability**. This opens a private advisory visible only to the maintainers. -To report a security issue, please email the maintainers with a description of the issue, the steps you took to create the issue, -affected versions, and, if known, mitigations for the issue. +Please include: -All support will be made on the best-effort basis, so please indicate the "urgency level" of the vulnerability as Critical, High, Medium or Low. +* a description of the issue and its impact; +* the version or commit hash affected, and the build configuration (compiler, `CMAKE_BUILD_TYPE`, any optional features such as `ENABLE_GLCACHE` / `ENABLE_LRB` / `ENABLE_3L_CACHE`); +* steps to reproduce — ideally a trace file or a generator for one, plus the exact command line; +* any known mitigations or a suggested fix. + +Because impact depends heavily on how libCacheSim is deployed, please indicate an urgency level of **Critical**, **High**, **Medium**, or **Low** and say why. + +## What to expect + +libCacheSim is maintained by a small group of researchers, so responses are best-effort rather than on a fixed schedule. We will acknowledge your report, tell you whether we consider it a vulnerability, and let you know when a fix lands. We are happy to credit you in the advisory unless you prefer otherwise. + +## Scope + +libCacheSim is a simulation and analysis library. It parses trace files, which are the main untrusted input: memory-safety bugs reachable from a malformed or malicious trace (in the trace readers, the CLI tools, or the cache implementations) are in scope. + +Out of scope: + +* crashes caused by deliberately invalid command-line arguments; +* resource exhaustion from legitimately large traces or cache sizes; +* issues in third-party code vendored under `libCacheSim/dataStructure/` or `libCacheSim/cache/eviction/{LHD,LRB,3LCache}/` — please report those upstream, though we appreciate a heads-up. + +For non-security bugs, please use the [issue tracker](https://github.com/1a1a11a/libCacheSim/issues). From c23b0efab5b543126e9c906ffdfc40208b5af297 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:26:27 +0000 Subject: [PATCH 04/33] chore: remove scratch files from the repo root Files no build, script or doc referenced: - test.c: a stale copy of the README library example, still pointing at data/trace.vscsi and referencing a quickstart.md that no longer exists. The README carries a correct, more complete version inline. - random/: a glib allocator microbenchmark and its pasted stdout. - doc/TODO, scripts/note: personal notes. - package-lock.json: an empty stub at the root with no dependencies; the real Node package is under libCacheSim-node/. .gitignore grouped by purpose, and its `.vscode/*` rule no longer contradicts the four .vscode/*.json files that are deliberately checked in as shared editor config. Added the traces traceConv and traceFilter drop into data/ when the docs are followed from the repository root. .editorconfig mirrors .clang-format (Google style, 2-space, 80 columns) so editors match CI without contributors configuring anything. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- .editorconfig | 31 +++++++++++ .gitignore | 44 +++++++++------ doc/TODO | 9 --- package-lock.json | 6 -- random/allocator.c | 122 ----------------------------------------- random/allocatorResult | 106 ----------------------------------- scripts/note | 9 --- test.c | 37 ------------- 8 files changed, 59 insertions(+), 305 deletions(-) create mode 100644 .editorconfig delete mode 100644 doc/TODO delete mode 100644 package-lock.json delete mode 100644 random/allocator.c delete mode 100644 random/allocatorResult delete mode 100644 scripts/note delete mode 100644 test.c diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..e4fade85f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,31 @@ +# https://editorconfig.org +# Mirrors .clang-format (Google style, 2-space indent, 80 columns). +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.{c,h,cc,cpp,hpp}] +indent_size = 2 +max_line_length = 80 + +[*.py] +indent_size = 4 + +[*.{yml,yaml,json,js,cmake}] +indent_size = 2 + +[CMakeLists.txt] +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.md] +# trailing double-space is a hard line break in Markdown +trim_trailing_whitespace = false diff --git a/.gitignore b/.gitignore index 620e8536b..cade71181 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,36 @@ -# 1a1a11a -__pycache__ -*deprecated* -*.DS_Store* -*.bak -*.clean -*.nogit* +# Build output +build *_build* +example/cacheSimulatorC/cmake-build-debug *.out -build +*.whl + +# Editors and IDEs .idea -example/cacheSimulatorC/cmake-build-debug +sftp-config.json +# the checked-in .vscode/*.json files are shared editor config; ignore the rest .vscode/* +!.vscode/c_cpp_properties.json +!.vscode/launch.json +!.vscode/settings.json +!.vscode/tasks.json + +# Caches and generated files +__pycache__ +*.cache/ +.lint-logs/ *.log +*.DS_Store* + +# Scratch and local data +*deprecated* +*.bak +*.clean +*.nogit* fig/ result/ data_large/ -# Chaos -sftp-config.json -# Clangd cache -*.cache/ -.lint-logs/ -# Python wheels -*.whl + +# Traces produced by traceConv/traceFilter when run against data/ +data/*.oracleGeneral +data/*.lcs.* diff --git a/doc/TODO b/doc/TODO deleted file mode 100644 index 32da87f65..000000000 --- a/doc/TODO +++ /dev/null @@ -1,9 +0,0 @@ - -1. add support for set-associative cache -2. add support for space management -3. mac compatibility -4. add support for cache clusters -5. add support for cache hierarchy -6. fix oracle trace gen -7. update the analysis scripts - diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 9aa734530..000000000 --- a/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "libCacheSim", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/random/allocator.c b/random/allocator.c deleted file mode 100644 index 8461b0940..000000000 --- a/random/allocator.c +++ /dev/null @@ -1,122 +0,0 @@ -// -// Created by Juncheng Yang on 6/9/20. -// - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define TIMEVAL_TO_USEC(tv) ((long long)(tv.tv_sec * 1000000 + tv.tv_usec)) -#define TIMEVAL_TO_SEC(tv) ((double)(tv.tv_sec + tv.tv_usec / 1000000.0)) -#define my_malloc(type) (type *)malloc(sizeof(type)) - -typedef void *(*allocator)(size_t); - -typedef void (*free_func)(void *); - -typedef void (*workload_func)(allocator, const char *); - -#define N (1024L * 1024 * 100) -#define ALIGN_SIZE 64 -static int alloc_size = 0; -static int alloc_type = 0; -static int workload_idx = 0; -static void *it[N]; - -static inline void g_slice_free2(void *mem) { g_slice_free1(alloc_size, mem); } - -static inline void *aligned_alloc2(size_t size) { - return aligned_alloc(ALIGN_SIZE, size); -} - -void print_rusage_diff(struct rusage r1, struct rusage r2) { - printf("****** CPU user time %.2lf s, sys time %.2lf s\n", - (TIMEVAL_TO_SEC(r2.ru_utime) - TIMEVAL_TO_SEC(r1.ru_utime)), - (TIMEVAL_TO_SEC(r2.ru_stime) - TIMEVAL_TO_SEC(r1.ru_stime))); - - printf( - "****** Mem RSS %.2lf MB, soft page fault %ld - hard page fault %ld, " - "voluntary context switches %ld - involuntary %ld\n", - (double)(r2.ru_maxrss - r1.ru_maxrss) / (1024.0), - (r2.ru_minflt - r1.ru_minflt), (r2.ru_majflt - r1.ru_majflt), - (r2.ru_nvcsw - r1.ru_nvcsw), (r2.ru_nivcsw - r1.ru_nivcsw)); -} - -void workload0(allocator alloc, const char *allocator_name) { - /* single size allocation */ - printf("%s alloc size %d, expected RSS %ld MiB\n", allocator_name, alloc_size, - alloc_size * (N / 1024 / 1024L)); - - for (int i = 0; i < N; i++) { - it[i] = alloc(alloc_size); - *(uint64_t *)(it[i]) = i; - } -} - -void workload1(allocator alloc, const char *allocator_name) { - /* single size allocation */ - printf("%s alloc size %d, expected RSS %ld MiB\n", allocator_name, alloc_size, - alloc_size * (N / 1024 / 1024L)); - for (int i = 0; i < N; i++) { - it[i] = alloc(alloc_size); - } -} - -void eval_alloc_perf() { - const char *allocator_names[] = {"malloc", "g_malloc", "g_slice_alloc", - "my_malloc"}; - const allocator allocators[] = {malloc, g_malloc, g_slice_alloc, - aligned_alloc2}; - const free_func free_funcs[] = {free, g_free, g_slice_free2, free}; - const workload_func workloads[] = {workload0, workload1}; - - struct rusage r_usage_before, r_usage_after; - getrusage(RUSAGE_SELF, &r_usage_before); - - workloads[workload_idx](allocators[alloc_type], allocator_names[alloc_type]); - - // for (int i = 0; i < N; i++) - // free_funcs[alloc_type](it[i]); - - getrusage(RUSAGE_SELF, &r_usage_after); - print_rusage_diff(r_usage_before, r_usage_after); - - // sleep(20); -} - -/** - * LD_PRELOAD=/home/jason/software/source/gperftools-2.7/.libs/libtcmalloc.so - * LD_PRELOAD=/usr/local/lib/libjemalloc.so - * LD_PRELOAD=/usr/lib/libhoard.so - * gcc allocator.c $(pkg-config --cflags --libs glib-2.0) -o binAllocatorEval; - * gcc allocator.c -ltcmalloc $(pkg-config --cflags --libs glib-2.0) -o - * binAllocatorEval; gcc allocator.c -ljemalloc $(pkg-config --cflags --libs - * glib-2.0) -o binAllocatorEval; gcc allocator.c -lhoard $(pkg-config --cflags - * --libs glib-2.0) -o binAllocatorEval; gcc allocator.c - * -L/home/jason/software/source/ptmalloc/ptmalloc.o $(pkg-config --cflags - * --libs glib-2.0) -o binAllocatorEval; - * - * - * for s in 4 8 16 32 64 128 256; do - * ./binAllocatorEval 0 $s 0 - * done - */ - -int main(int argc, char *argv[]) { - if (argc != 4) { - printf("usage %s alloc_type alloc_size workload\n", argv[0]); - exit(1); - } - alloc_type = atoi(argv[1]); - alloc_size = atoi(argv[2]); - workload_idx = atoi(argv[3]); - eval_alloc_perf(); - return 0; -} diff --git a/random/allocatorResult b/random/allocatorResult deleted file mode 100644 index c15624571..000000000 --- a/random/allocatorResult +++ /dev/null @@ -1,106 +0,0 @@ - - -compared to malloc, g_malloc is 10% slower, g_new is 2x slower, g_new0 is similar to g_new (like 10% slower) - - -####################### workload: Single size allocation ######################### -g_slice_alloc alloc size 4, expected RSS 400 MiB -****** CPU user time 1.79 s, sys time 0.52 s -****** Mem RSS 2768.95 MB, soft page fault 708933 - hard page fault 0, voluntary context switches 0 - involuntary 88 -g_slice_alloc alloc size 8, expected RSS 800 MiB -****** CPU user time 1.83 s, sys time 0.49 s -****** Mem RSS 2768.96 MB, soft page fault 708932 - hard page fault 0, voluntary context switches 0 - involuntary 108 -g_slice_alloc alloc size 16, expected RSS 1600 MiB -****** CPU user time 1.81 s, sys time 0.50 s -****** Mem RSS 2768.95 MB, soft page fault 708935 - hard page fault 0, voluntary context switches 0 - involuntary 96 -g_slice_alloc alloc size 32, expected RSS 3200 MiB -****** CPU user time 2.02 s, sys time 0.77 s -****** Mem RSS 4456.88 MB, soft page fault 1141049 - hard page fault 0, voluntary context switches 0 - involuntary 116 -g_slice_alloc alloc size 64, expected RSS 6400 MiB -****** CPU user time 2.34 s, sys time 1.32 s -****** Mem RSS 7626.48 MB, soft page fault 1952437 - hard page fault 0, voluntary context switches 0 - involuntary 5 -g_slice_alloc alloc size 128, expected RSS 12800 MiB -****** CPU user time 3.14 s, sys time 2.40 s -****** Mem RSS 14453.16 MB, soft page fault 3700068 - hard page fault 0, voluntary context switches 0 - involuntary 240 -g_slice_alloc alloc size 256, expected RSS 25600 MiB -****** CPU user time 4.09 s, sys time 5.04 s -****** Mem RSS 28106.50 MB, soft page fault 7195330 - hard page fault 0, voluntary context switches 0 - involuntary 363 - - -############## default malloc -malloc alloc size 4, expected RSS 400 MiB -****** CPU user time 2.08 s, sys time 0.77 s -****** Mem RSS 3999.57 MB, soft page fault 1024005 - hard page fault 0, voluntary context switches 0 - involuntary 4 -malloc alloc size 8, expected RSS 800 MiB -****** CPU user time 2.12 s, sys time 0.73 s -****** Mem RSS 3999.56 MB, soft page fault 1024005 - hard page fault 0, voluntary context switches 0 - involuntary 115 -malloc alloc size 16, expected RSS 1600 MiB -****** CPU user time 2.21 s, sys time 0.68 s -****** Mem RSS 3999.57 MB, soft page fault 1024006 - hard page fault 0, voluntary context switches 0 - involuntary 3 -malloc alloc size 32, expected RSS 3200 MiB -****** CPU user time 2.36 s, sys time 0.94 s -****** Mem RSS 5599.61 MB, soft page fault 1433604 - hard page fault 0, voluntary context switches 0 - involuntary 162 -malloc alloc size 64, expected RSS 6400 MiB -****** CPU user time 2.54 s, sys time 1.57 s -****** Mem RSS 8799.54 MB, soft page fault 2252804 - hard page fault 0, voluntary context switches 0 - involuntary 5 -malloc alloc size 128, expected RSS 12800 MiB -****** CPU user time 3.92 s, sys time 2.85 s -****** Mem RSS 15199.71 MB, soft page fault 3891205 - hard page fault 0, voluntary context switches 0 - involuntary 314 -malloc alloc size 256, expected RSS 25600 MiB -****** CPU user time 6.37 s, sys time 5.40 s -****** Mem RSS 27999.70 MB, soft page fault 7168004 - hard page fault 0, voluntary context switches 1 - involuntary 14 - -################ tcmalloc -malloc alloc size 4, expected RSS 400 MiB -****** CPU user time 1.56 s, sys time 0.31 s -****** Mem RSS 1605.62 MB, soft page fault 410855 - hard page fault 0, voluntary context switches 0 - involuntary 91 -malloc alloc size 8, expected RSS 800 MiB -****** CPU user time 1.61 s, sys time 0.23 s -****** Mem RSS 1605.62 MB, soft page fault 410855 - hard page fault 0, voluntary context switches 0 - involuntary 3 -malloc alloc size 16, expected RSS 1600 MiB -****** CPU user time 1.61 s, sys time 0.46 s -****** Mem RSS 2412.47 MB, soft page fault 617420 - hard page fault 0, voluntary context switches 0 - involuntary 99 -malloc alloc size 32, expected RSS 3200 MiB -****** CPU user time 1.83 s, sys time 0.70 s -****** Mem RSS 4022.28 MB, soft page fault 1029523 - hard page fault 0, voluntary context switches 0 - involuntary 107 -malloc alloc size 64, expected RSS 6400 MiB -****** CPU user time 2.24 s, sys time 1.25 s -****** Mem RSS 7245.79 MB, soft page fault 1854755 - hard page fault 0, voluntary context switches 0 - involuntary 189 -malloc alloc size 128, expected RSS 12800 MiB -****** CPU user time 3.18 s, sys time 2.27 s -****** Mem RSS 13690.93 MB, soft page fault 3504706 - hard page fault 0, voluntary context switches 0 - involuntary 214 -malloc alloc size 256, expected RSS 25600 MiB -****** CPU user time 4.68 s, sys time 4.52 s -****** Mem RSS 26583.32 MB, soft page fault 6805120 - hard page fault 0, voluntary context switches 0 - involuntary 378 - -################ jemalloc -malloc alloc size 4, expected RSS 400 MiB -****** CPU user time 2.19 s, sys time 0.24 s -****** Mem RSS 1627.56 MB, soft page fault 422807 - hard page fault 0, voluntary context switches 0 - involuntary 123 -malloc alloc size 8, expected RSS 800 MiB -****** CPU user time 2.08 s, sys time 0.34 s -****** Mem RSS 1627.58 MB, soft page fault 422806 - hard page fault 0, voluntary context switches 0 - involuntary 3 -malloc alloc size 16, expected RSS 1600 MiB -****** CPU user time 2.41 s, sys time 0.37 s -****** Mem RSS 2454.10 MB, soft page fault 640808 - hard page fault 0, voluntary context switches 0 - involuntary 132 -malloc alloc size 32, expected RSS 3200 MiB -****** CPU user time 2.61 s, sys time 0.82 s -****** Mem RSS 4107.21 MB, soft page fault 1076808 - hard page fault 0, voluntary context switches 0 - involuntary 6 -malloc alloc size 64, expected RSS 6400 MiB -****** CPU user time 3.32 s, sys time 1.49 s -****** Mem RSS 7413.42 MB, soft page fault 1948810 - hard page fault 0, voluntary context switches 0 - involuntary 246 -malloc alloc size 128, expected RSS 12800 MiB -****** CPU user time 4.88 s, sys time 2.64 s -****** Mem RSS 14026.05 MB, soft page fault 3692819 - hard page fault 0, voluntary context switches 0 - involuntary 10 -malloc alloc size 256, expected RSS 25600 MiB -****** CPU user time 8.08 s, sys time 5.06 s -****** Mem RSS 27250.97 MB, soft page fault 7180834 - hard page fault 0, voluntary context switches 0 - involuntary 554 - - -################ hoard - - - - -####################### workload: Single size allocation ######################### - diff --git a/scripts/note b/scripts/note deleted file mode 100644 index 5688d7c90..000000000 --- a/scripts/note +++ /dev/null @@ -1,9 +0,0 @@ -## How to run Caffeine simulator -``` -git clone https://github.com/ben-manes/caffeine.git -./gradlew build -export GRADLE_OPTS="-Xmx204800m" -# modify simulator/src/main/resources/reference.conf, the lirs format is txt with only the object id -./gradlew run simulator:run - -``` diff --git a/test.c b/test.c deleted file mode 100644 index d310ed684..000000000 --- a/test.c +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include -#include - -int main(int argc, char *argv[]) { - /* open trace, see quickstart.md for opening csv and binary trace */ - reader_t *reader = open_trace("../data/trace.vscsi", VSCSI_TRACE, NULL); - - /* create a container for reading from trace */ - request_t *req = new_request(); - - /* create a LRU cache */ - common_cache_params_t cc_params = default_common_cache_params(); - cc_params.cache_size = 1024 * 1024U; - cache_t *cache = LRU_init(cc_params, NULL); - - /* counters */ - uint64_t req_byte = 0, miss_byte = 0; - - /* loop through the trace */ - while (read_one_req(reader, req) == 0) { - if (cache->get(cache, req) == false) { - miss_byte += req->obj_size; - } - req_byte += req->obj_size; - } - - /* cleaning */ - close_trace(reader); - free_request(req); - cache->cache_free(cache); - - return 0; -} - -// compile with the following -// gcc test.c $(pkg-config --cflags --libs libCacheSim glib-2.0) -o test.out From 47f91ef1b9030522900ffbfb435214d12b3e79ba Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:26:37 +0000 Subject: [PATCH 05/33] ci: replace archived release action and harden checkouts - npm-release.yml used actions/create-release@v1, which GitHub archived in 2021. Replaced with `gh release create`, which the runners already have and which the preceding step already uses. This also makes the job's release_created output real; the archived action never emitted it, so the value was always empty. - code-quality.yml was still on actions/checkout@v3 while every other workflow used v4. - persist-credentials: false on all checkouts, so the job token is not left in .git/config for later steps (OpenSSF Scorecard flags this, and the repo publishes a Scorecard badge). - libCacheSim-node/package.json homepage pointed into tree/main, but there is no main branch, so the link 404s from the npm page. Workflow YAML validated and the new release step dry-run with gh stubbed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- .github/workflows/build.yml | 4 +++ .github/workflows/code-quality.yml | 3 ++- .github/workflows/codeql-analysis.yml | 2 ++ .github/workflows/npm-release.yml | 35 +++++++++++++++------------ libCacheSim-node/package.json | 2 +- 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fac10c2b..a2214e6f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,8 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Prepare run: bash scripts/install_dependency.sh - name: Configure CMake @@ -29,6 +31,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Prepare run: bash scripts/install_dependency.sh - name: Configure CMake with LSan diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index f3f21b2f5..264bfebb2 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -10,9 +10,10 @@ jobs: name: Code Quality Checks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch all history for proper git diff + persist-credentials: false - name: Install dependencies run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f8a7ac829..e609c1072 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,6 +25,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Initialize CodeQL uses: github/codeql-action/init@v4 diff --git a/.github/workflows/npm-release.yml b/.github/workflows/npm-release.yml index 065ade935..26f3d65da 100644 --- a/.github/workflows/npm-release.yml +++ b/.github/workflows/npm-release.yml @@ -29,6 +29,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Synchronize Node.js binding version run: | @@ -57,29 +59,29 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # actions/create-release was archived by GitHub in 2021; the gh CLI is + # preinstalled on the runners and is already used by the step above. - name: Create GitHub Release id: release if: steps.check_release.outputs.exists == 'false' - uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: v${{ steps.package.outputs.version }} - release_name: Release v${{ steps.package.outputs.version }} - body: | - Release v${{ steps.package.outputs.version }} + VERSION: ${{ steps.package.outputs.version }} + run: | + gh release create "v${VERSION}" \ + --title "Release v${VERSION}" \ + --notes "Release v${VERSION} - ## Installation - ```bash - npm install libcachesim-node - ``` + ## Installation + \`\`\`bash + npm install libcachesim-node + \`\`\` - ## Supported Platforms - - Linux x64 + ## Supported Platforms + - Linux x64 - Pre-compiled binaries are automatically downloaded during installation. - draft: false - prerelease: false + Pre-compiled binaries are automatically downloaded during installation." + echo "release_created=true" >> "$GITHUB_OUTPUT" build-and-publish: if: github.event_name == 'release' @@ -92,6 +94,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: + persist-credentials: false fetch-depth: 0 - name: Synchronize Node.js binding version @@ -157,6 +160,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Synchronize Node.js binding version run: | diff --git a/libCacheSim-node/package.json b/libCacheSim-node/package.json index fc06593a6..5187ec31b 100644 --- a/libCacheSim-node/package.json +++ b/libCacheSim-node/package.json @@ -27,7 +27,7 @@ "type": "git", "url": "https://github.com/1a1a11a/libCacheSim" }, - "homepage": "https://github.com/1a1a11a/libCacheSim/tree/main/libCacheSim-node", + "homepage": "https://github.com/1a1a11a/libCacheSim/tree/develop/libCacheSim-node", "bugs": { "url": "https://github.com/1a1a11a/libCacheSim/issues" }, From c9b6a1a1a457a90a13fe83e34c637dafd0d19b35 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:37:51 +0000 Subject: [PATCH 06/33] fix: unbreak macOS build and correct the reader examples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit macOS CI has been red since the runner image moved to Xcode 26.6 — including on docs-only commits to develop, so this predates this branch. clang now rejects `UINT64_MAX * sample_rate` and `... / UINT64_MAX` under -Werror, because UINT64_MAX has no exact double representation (-Wimplicit-const-int-float-conversion). Made both widenings explicit in mrcProfiler; the arithmetic is unchanged, and both SHARDS paths were re-run to confirm identical output. doc/advanced_lib.md's reader examples were wrong in three ways, which repointing them at the sample trace made visible: - obj_id_field was 6, but cloudPhysicsIO.csv is version,time,op,size,lbn so the id is field 5. Copying it verbatim gave every row the same default id and meaningless results. - has_header was FALSE for a file that has a header, and obj_id_is_num was unset on a numeric id column. - the binary example used .binary_fmt, which is not a field (.binary_fmt_str), with "<3I2H2Q" — the format parser has no repeat counts, so it errored with "unknown format '3'". Expanded to " Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/advanced_lib.md | 17 ++++++++++++----- libCacheSim/cache/eviction/SLRU.c | 11 +++++++++++ libCacheSim/mrcProfiler/mrcProfiler.cpp | 10 +++++++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/doc/advanced_lib.md b/doc/advanced_lib.md index 1e0db26b0..4b059b67d 100644 --- a/doc/advanced_lib.md +++ b/doc/advanced_lib.md @@ -113,16 +113,23 @@ open_trace(data_path, PLAIN_TXT_TRACE, NULL); ``` ##### Setup a csv reader +The fields are 1-indexed and must match the trace. The sample `data/cloudPhysicsIO.csv` has the header `version,time,op,size,lbn`, so time is field 2, size is field 4, and the object id is field 5. Set `obj_id_is_num` when the id column holds numbers, otherwise the ids are hashed. + ```c -reader_init_param_t init_params_csv = - {.delimiter=',', .time_field=2, .obj_id_field=6, .obj_size_field=4, .has_header=FALSE}; -reader_t *reader_csv_c = open_trace("data/cloudPhysicsIO.csv", CSV_TRACE, &init_params_csv); +reader_init_param_t init_params_csv = {.delimiter = ',', + .time_field = 2, + .obj_size_field = 4, + .obj_id_field = 5, + .obj_id_is_num = true, + .has_header = true}; +reader_t *reader_csv = open_trace("data/cloudPhysicsIO.csv", CSV_TRACE, &init_params_csv); ``` ##### Setup a binary reader ```c -reader_init_param_t init_params_bin = {.binary_fmt="<3I2H2Q", .obj_size_field=2, .obj_id_field=6, }; -reader_t *reader_bin_l = setup_reader("data/cloudPhysicsIO.vscsi", BIN_TRACE, &init_params_bin); +reader_init_param_t init_params_bin = { + .binary_fmt_str = " 2) { ERROR("param parsing error, find string \"%s\" after number\n", end); } + /* n_seg divides the cache size and the reported percentages */ + if (params->n_seg < 1 || params->n_seg > SLRU_MAX_N_SEG) { + ERROR("n-seg must be between 1 and %d, got %d\n", SLRU_MAX_N_SEG, + params->n_seg); + } } else if (strcasecmp(key, "seg-size") == 0) { int n_seg = 0; int64_t seg_size_sum = 0; int64_t seg_size_array[SLRU_MAX_N_SEG]; char *v = strsep((char **)&value, ":"); while (v != NULL) { + if (n_seg >= SLRU_MAX_N_SEG) { + ERROR("seg-size accepts at most %d segments\n", SLRU_MAX_N_SEG); + } seg_size_array[n_seg++] = (int64_t)strtol(v, &end, 0); seg_size_sum += seg_size_array[n_seg - 1]; v = strsep((char **)&value, ":"); } + if (n_seg < 1 || seg_size_sum <= 0) { + ERROR("seg-size needs at least one segment with a positive size\n"); + } params->n_seg = n_seg; params->lru_max_n_bytes = calloc(params->n_seg, sizeof(int64_t)); for (int i = 0; i < n_seg; i++) { diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index 1e5da9d6e..ff92a3b77 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -95,7 +95,10 @@ void mrcProfiler::MRCProfilerSHARDS::fixed_sample_rate_run() { double sample_rate = params_.shards_params.sample_rate; std::vector local_hit_cnt_vec(mrc_size_vec.size(), 0); std::vector local_hit_size_vec(mrc_size_vec.size(), 0); - uint64_t sample_max = UINT64_MAX * sample_rate; + /* UINT64_MAX has no exact double representation, so make the widening + * explicit; clang errors on the implicit form under -Werror */ + uint64_t sample_max = + static_cast(static_cast(UINT64_MAX) * sample_rate); if (sample_rate == 1) { INFO("sample_rate is 1, no need to sample\n"); sample_max = UINT64_MAX; @@ -212,8 +215,9 @@ void mrcProfiler::MRCProfilerSHARDS::fixed_sample_size_run() { if (!min_value_map.full()) { sample_rate = 1.0; // still 100% sample rate } else { - sample_rate = min_value_map.get_max_value() * 1.0 / - UINT64_MAX; // adjust the sample rate + sample_rate = + min_value_map.get_max_value() * 1.0 / + static_cast(UINT64_MAX); // adjust the sample rate } sampled_cnt += 1.0 / sample_rate; From 534ae73a283be5db3d18e321ecc67e70029a63ca Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:50:04 +0000 Subject: [PATCH 07/33] fix: declare the node package's real license MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libCacheSim-node/package.json said "MIT" while the project is GPL-3.0 and binding.gyp links vendor/liblibCacheSim.a statically, making the addon a derivative work. npm has been publishing it under the wrong license. The package README said "MIT License - see the LICENSE file", where that file is the repository's GPLv3 — the two contradicted each other on the same page. Now GPL-3.0-only in both, matching the LICENSE file. Using -only rather than -or-later because the repo ships the plain GPLv3 text with no "or (at your option) any later version" notice in any source file; switch to GPL-3.0-or-later if that was the intent. CITATION.cff uses the same identifier so the two agree. Also fixed the package README's link to the Python bindings, which pointed at libCacheSim/pyBindings — a path that does not exist. They live in the cacheMon/libCacheSim-python repository. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- CITATION.cff | 2 +- libCacheSim-node/README.md | 4 ++-- libCacheSim-node/package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 7674c21d7..e3f28b4b3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -20,7 +20,7 @@ keywords: - eviction algorithm - miss ratio curve - trace analysis -license: GPL-3.0 +license: GPL-3.0-only preferred-citation: type: conference-paper title: FIFO Queues Are All You Need for Cache Eviction diff --git a/libCacheSim-node/README.md b/libCacheSim-node/README.md index ac07a5e24..b1ce44aac 100644 --- a/libCacheSim-node/README.md +++ b/libCacheSim-node/README.md @@ -202,12 +202,12 @@ Contributions are welcome! Please see the main [libCacheSim repository](https:// ## License -MIT License - see the LICENSE file for details. +GPL-3.0 - see the [LICENSE](https://github.com/1a1a11a/libCacheSim/blob/develop/LICENSE) file for details. This addon links libCacheSim statically, so the same terms apply to it. ## Related Projects - [libCacheSim](https://github.com/1a1a11a/libCacheSim) - The core C library -- [libCacheSim Python bindings](https://github.com/1a1a11a/libCacheSim/tree/develop/libCacheSim/pyBindings) - Python interface +- [libCacheSim Python bindings](https://github.com/cacheMon/libCacheSim-python) - Python interface ## Citation diff --git a/libCacheSim-node/package.json b/libCacheSim-node/package.json index 5187ec31b..6b4d50cdc 100644 --- a/libCacheSim-node/package.json +++ b/libCacheSim-node/package.json @@ -21,7 +21,7 @@ "libcachesim" ], "author": "Murphy Tian", - "license": "MIT", + "license": "GPL-3.0-only", "description": "Node.js bindings for libCacheSim - A high-performance cache simulator and analysis library supporting LRU, FIFO, S3-FIFO, Sieve and other caching algorithms", "repository": { "type": "git", From 896eda8ba29db578e1e37cbd7a06a978e07d9903 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:50:26 +0000 Subject: [PATCH 08/33] docs: make the Read the Docs build work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .readthedocs.yaml pointed sphinx at docs/conf.py. There is no docs/ directory, no conf.py and no .rst anywhere in the repo — the docs are the Markdown in doc/ — so every Read the Docs build failed immediately. Wired up Sphinx over the existing Markdown with MyST, so the sources stay readable on GitHub and need no duplication: - doc/conf.py, doc/index.md (toctree entry point), doc/requirements.txt. RTD installed the root requirements.txt (numpy/matplotlib/pandas) for a docs build that needs none of it; it now installs only doc deps. - The Markdown links into the repository as "/libCacheSim/..." and "../README.md", which resolve on github.com but not on a docs site. A source-read hook rewrites those to blob URLs while keeping links between doc pages local, so the sidebar, search and anchors work. It has to run before MyST resolves links, otherwise each one is reported as a missing cross-reference. - ```mermaid fences render as diagrams instead of failing to highlight. Set fail_on_warning, having got the build to zero warnings from 96: - doc/API.md had no headings at all, so it could not appear in a toctree, and it had drifted from the headers: reader_init_param_t was missing most of its fields and named the binary format binary_fmt (it is binary_fmt_str), get_num_of_req returned uint64_t, the simulator returned a sim_res_t that no longer exists, and the file trailed off at a "profiler:" heading with nothing under it. Rewritten against the current headers; every symbol checked to exist. - advanced_lib.md documented the same stale simulator API, including a "reader_t reader*" typo, plus heading levels that skipped from H2 to H4. - performance.md was two empty sections and one bullet. Filled in with the knobs that exist — tcmalloc, USE_HUGEPAGE, binary traces, the threading and sizing flags — each verified against the CLI and CMake. - install.md, quickstart_traceAnalyzer.md and quickstart_traceUtils.md started at H2, so promoted their heading hierarchies (skipping fenced code, which contains #-comments). - A massif report was tagged as shell and failed to lex. Verified with `sphinx-build -W`: 13 pages, no warnings. Checked the generated HTML rewrites repo links, keeps intra-doc links local, renders the mermaid diagrams, and leaves no root-absolute href behind. All markdown links still resolve for the GitHub view. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- .gitignore | 1 + .readthedocs.yaml | 13 +- doc/API.md | 305 ++++++++++++++++++++------------ doc/advanced_lib.md | 82 +++++---- doc/advanced_lib_extend.md | 2 +- doc/conf.py | 93 ++++++++++ doc/index.md | 51 ++++++ doc/install.md | 16 +- doc/memory_usage_profiling.md | 2 +- doc/performance.md | 61 ++++++- doc/quickstart_traceAnalyzer.md | 30 ++-- doc/quickstart_traceUtils.md | 8 +- doc/requirements.txt | 6 + 13 files changed, 479 insertions(+), 191 deletions(-) create mode 100644 doc/conf.py create mode 100644 doc/index.md create mode 100644 doc/requirements.txt diff --git a/.gitignore b/.gitignore index cade71181..161e23b57 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ sftp-config.json !.vscode/tasks.json # Caches and generated files +doc/_build/ __pycache__ *.cache/ .lint-logs/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 87c2cd65e..7b0379bf2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,13 +11,14 @@ build: tools: python: "3.11" -# Build documentation in the docs/ directory with Sphinx +# The documentation sources are the Markdown files in doc/, rendered with MyST. sphinx: - configuration: docs/conf.py + configuration: doc/conf.py + # The build is warning-free; keep it that way, since a broken cross-reference + # is otherwise easy to miss. + fail_on_warning: true -# We recommend specifying your dependencies to enable reproducible builds: -# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# Docs-only dependencies; the root requirements.txt is for the analysis scripts. python: install: - - requirements: requirements.txt - + - requirements: doc/requirements.txt diff --git a/doc/API.md b/doc/API.md index 9a6011d71..8f2112aa2 100644 --- a/doc/API.md +++ b/doc/API.md @@ -1,148 +1,225 @@ -traceReader: -```C -typedef struct { - int time_field; - int obj_id_field; - int obj_size_field; - int op_field; - int ttl_field; +# C API reference - // csv reader - gboolean has_header; - char delimiter; +The public C API is exposed through a single header: - // binary reader - char binary_fmt[MAX_BIN_FMT_STR_LEN]; -} reader_init_param_t; +```c +#include +``` + +Compile against it with pkg-config: + +```bash +gcc your_program.c $(pkg-config --cflags --libs libCacheSim glib-2.0) -o your_program -lm -lzstd +``` -typedef struct reader { - char *mapped_file; /* mmap the file, this should not change during runtime */ - uint64_t mmap_offset; +See [advanced_lib.md](advanced_lib.md) for a walkthrough and [the example folder](/example) for complete programs. The declarations below are the commonly used subset; the headers under [libCacheSim/include/libCacheSim/](/libCacheSim/include/libCacheSim/) are authoritative. - FILE *file; - size_t file_size; +--- - trace_type_e trace_type; /* possible types see trace_type_t */ +## Reading traces - size_t item_size; /* the size of one record, used to - * locate the memory location of next element, - * when used in vscsiReaser and binaryReader, - * it is a const value, - * when it is used in plainReader or csvReader, - * it is the size of last record, it does not - * include LFCR or \0 */ +### Opening a trace - uint64_t n_total_req; /* number of requests in the trace */ - uint64_t n_uniq_obj; /* number of objects in the trace */ +`reader_init_param_t` describes how to interpret a trace. Field indices are 1-based, and `0` means the field is absent. Start from `default_reader_init_params()` rather than zero-initializing, so the defaults for delimiter, sampling, and the "was this set by the user" flags are correct. - char trace_path[MAX_FILE_PATH_LEN]; - reader_init_param_t init_params; +```c +typedef struct { + bool ignore_obj_size; + bool ignore_size_zero_req; + bool obj_id_is_num; + bool obj_id_is_num_set; // whether the user passed this parameter + int64_t cap_at_n_req; // process at most n requests + + int32_t time_field; + int32_t obj_id_field; + int32_t obj_size_field; + int32_t obj_cost_field; + int32_t op_field; + int32_t ttl_field; + int32_t cnt_field; + int32_t tenant_field; + int32_t next_access_vtime_field; + + int32_t n_feature_fields; + int32_t feature_fields[N_MAX_FEATURES]; + + // block cache; breaks a large request into per-block requests + int32_t block_size; - void *reader_params; - void *other_params; /* currently not used */ + // csv reader + bool has_header; + bool has_header_set; // false alone cannot distinguish "unset" + char delimiter; - gint ver; + // skip metadata at the start of a binary trace + ssize_t trace_start_offset; - bool cloned; // true if this is a cloned reader, else false + // binary reader, a Python struct format string + char *binary_fmt_str; -} reader_t; + sampler_t *sampler; +} reader_init_param_t; + +static inline reader_init_param_t default_reader_init_params(void); /** - * setup the reader struct for reading trace - * @param trace_path + * open a trace for reading; the reader must be released with close_trace() * @param trace_type CSV_TRACE, PLAIN_TXT_TRACE, BIN_TRACE, VSCSI_TRACE, - * TWR_BIN_TRACE - * @param setup_params - * @return a pointer to reader_t struct, the returned reader needs to be - * explicitly closed by calling close_reader or close_trace + * ORACLE_GENERAL_TRACE, TWR_BIN_TRACE, LCS_TRACE, ... */ -reader_t *setup_reader(const char *trace_path, const trace_type_e trace_type, - const reader_init_param_t *const reader_init_param); +reader_t *setup_reader(const char *trace_path, trace_type_e trace_type, + const reader_init_param_t *reader_init_param); + +/* same function as setup_reader, and the more commonly used name */ +static inline reader_t *open_trace(const char *path, trace_type_e type, + const reader_init_param_t *reader_init_param); +``` -/* this is the same function as setup_reader */ -static inline reader_t * -open_trace(const char *path, const trace_type_e type, - const reader_init_param_t *const reader_init_param) { - return setup_reader(path, type, reader_init_param); -} +Object ids are hashed unless you set `obj_id_is_num`, which you should do when the id field holds numbers. -/** - * read one request from reader, and store it in the pre-allocated request_t req - * @param reader - * @param req - */ -uint64_t get_num_of_req(reader_t *const reader); +### Iterating over requests -/** - * as the name suggests - * @param reader - * @return - */ -static inline trace_type_e get_trace_type(const reader_t *const reader) { - return reader->trace_type; -} +```c +/* read one request into the pre-allocated req; returns 0 on success, + * 1 at end of trace */ +int read_one_req(reader_t *reader, request_t *req); -/** - * read one request from reader/trace, stored the info in pre-allocated req - * @param reader - * @param req - * return 0 on success and 1 if reach end of trace - */ -int read_one_req(reader_t *const reader, request_t *const req); +/* number of requests in the trace */ +int64_t get_num_of_req(reader_t *reader); -/** - * reset reader, so we can read from the beginning - * @param reader - */ -void reset_reader(reader_t *const reader); +static inline trace_type_e get_trace_type(const reader_t *reader); +static inline bool obj_id_is_num(const reader_t *reader); -/** - * close reader and release resources - * @param reader - * @return - */ -int close_reader(reader_t *const reader); +/* rewind so the trace can be read again */ +void reset_reader(reader_t *reader); -/** - * clone a reader, mostly used in multithreading - * @param reader - * @return - */ -reader_t *clone_reader(const reader_t *const reader); +/* clone a reader; the usual way to feed one trace to several threads */ +reader_t *clone_reader(const reader_t *reader); +int close_reader(reader_t *reader); +static inline int close_trace(reader_t *reader); ``` -cache and cacheAlgo: +Positioning helpers, used mostly by the analysis tools: -```C -static inline request_t *new_request(); -static inline void copy_request(request_t *req_dest, request_t *req_src); -static inline request_t *clone_request(request_t *req); +```c +void read_first_req(reader_t *reader, request_t *req); +void read_last_req(reader_t *reader, request_t *req); +int skip_n_req(reader_t *reader, int N); +int go_back_one_req(reader_t *reader); +void reader_set_read_pos(reader_t *reader, double pos); /* pos in [0, 1] */ +``` + +--- + +## Requests + +A `request_t` is the container `read_one_req()` fills in. Allocate one up front and reuse it for the whole trace. + +```c +static inline request_t *new_request(void); +static inline void copy_request(request_t *req_dest, const request_t *req_src); +static inline request_t *clone_request(const request_t *req); static inline void free_request(request_t *req); -static inline void print_request(request_t *req); +static inline void print_request(const request_t *req); ``` -simulator: -```C -sim_res_t * -simulate_at_multi_sizes(reader_t *const reader, - const cache_t *const cache, - const gint num_of_sizes, - const guint64 *const cache_sizes, - reader_t *const warmup_reader, - const double warmup_perc, - const gint num_of_threads); - - -sim_res_t * -simulate_at_multi_sizes_with_step_size(reader_t *const reader_in, - const cache_t *const cache_in, - const guint64 step_size, - reader_t *const warmup_reader, - const double warmup_perc, - const gint num_of_threads); +The fields you normally read are `obj_id`, `obj_size`, `clock_time`, `next_access_vtime` (oracle traces only), and `obj_cost`. + +--- + +## Caches + +Every eviction algorithm exposes an `_init` function taking the common parameters plus an optional algorithm-specific parameter string — the same string `cachesim` takes with `-e`. + +```c +typedef struct { + uint64_t cache_size; + uint64_t default_ttl; + int32_t hashpower; + bool consider_obj_metadata; +} common_cache_params_t; + +common_cache_params_t default_common_cache_params(void); + +cache_t *LRU_init(common_cache_params_t ccache_params, + const char *cache_specific_params); +/* ... and FIFO_init, ARC_init, S3FIFO_init, Sieve_init, and the rest; + * see libCacheSim/include/libCacheSim/evictionAlgo.h */ ``` +A `cache_t` is used through its function pointers: + +```c +/* the whole interface: lookup plus on-demand insert and evict. + * returns true on a cache hit */ +bool (*get)(cache_t *, const request_t *); + +/* look up without the insert/evict; update_cache controls whether the + * lookup also updates state such as recency */ +cache_obj_t *(*find)(cache_t *, const request_t *, bool update_cache); + +bool (*can_insert)(cache_t *, const request_t *); +cache_obj_t *(*insert)(cache_t *, const request_t *); + +/* which object would be evicted, without evicting it */ +cache_obj_t *(*to_evict)(cache_t *, const request_t *); +void (*evict)(cache_t *, const request_t *); + +/* user-triggered removal; eviction should go through evict instead */ +bool (*remove)(cache_t *, obj_id_t); +void (*cache_free)(cache_t *); +``` + +Most programs only need `get()`. See [advanced_lib_extend.md](advanced_lib_extend.md) to implement a new algorithm. + +--- + +## Simulator + +Rather than driving the loop yourself, you can hand a trace and a cache to the simulator, which parallelizes across cache sizes or across caches. + +```c +/* one cache, many sizes */ +cache_stat_t *simulate_at_multi_sizes(reader_t *reader, const cache_t *cache, + int num_of_sizes, + const uint64_t *cache_sizes, + reader_t *warmup_reader, + double warmup_frac, int warmup_sec, + int num_of_threads, bool use_random_seed); + +/* one cache, sizes at a fixed step up to the working set size */ +cache_stat_t *simulate_at_multi_sizes_with_step_size( + reader_t *reader_in, const cache_t *cache_in, uint64_t step_size, + reader_t *warmup_reader, double warmup_frac, int warmup_sec, + int num_of_threads, bool use_random_seed); + +/* many caches, each at its own configured size */ +cache_stat_t *simulate_with_multi_caches( + reader_t *reader, cache_t *caches[], int num_of_caches, + reader_t *warmup_reader, double warmup_frac, int warmup_sec, + int num_of_threads, bool free_cache_when_finish, bool use_random_seed); +``` + +Each returns an array with one `cache_stat_t` per simulation, which the caller frees: + +```c +typedef struct { + int64_t n_warmup_req; + int64_t n_req; + int64_t n_req_byte; + double n_req_cost; + int64_t n_miss; + int64_t n_miss_byte; + double n_miss_cost; + + int64_t n_obj; + int64_t occupied_byte; + int64_t cache_size; + float sampler_ratio; + /* ... */ +} cache_stat_t; +``` -profiler: +Object miss ratio is `n_miss / n_req`, and byte miss ratio is `n_miss_byte / n_req_byte`. diff --git a/doc/advanced_lib.md b/doc/advanced_lib.md index 4b059b67d..42f280a19 100644 --- a/doc/advanced_lib.md +++ b/doc/advanced_lib.md @@ -107,12 +107,12 @@ cache->to_evict(cache, req); There are mostly three APIs related to readers, `open_trace`, `close_trace`, `read_one_req`, let's take a look how they work. -##### Setup a txt reader (trace can only contain request id) +#### Setup a txt reader (trace can only contain request id) ```c open_trace(data_path, PLAIN_TXT_TRACE, NULL); ``` -##### Setup a csv reader +#### Setup a csv reader The fields are 1-indexed and must match the trace. The sample `data/cloudPhysicsIO.csv` has the header `version,time,op,size,lbn`, so time is field 2, size is field 4, and the object id is field 5. Set `obj_id_is_num` when the id column holds numbers, otherwise the ids are hashed. ```c @@ -125,7 +125,7 @@ reader_init_param_t init_params_csv = {.delimiter = ',', reader_t *reader_csv = open_trace("data/cloudPhysicsIO.csv", CSV_TRACE, &init_params_csv); ``` -##### Setup a binary reader +#### Setup a binary reader ```c reader_init_param_t init_params_bin = { .binary_fmt_str = "cache_size // it runs cache->cache_size/step_size simulations -sim_res_t * -simulate_at_multi_sizes_with_step_size(reader_t *reader, - cache_t *cache, - uint64_t step_size, - reader_t *warmup_reader, - double warmup_perc, - int num_of_threads); +cache_stat_t *simulate_at_multi_sizes_with_step_size(reader_t *reader_in, + const cache_t *cache_in, + uint64_t step_size, + reader_t *warmup_reader, + double warmup_frac, + int warmup_sec, + int num_of_threads, + bool use_random_seed); // simulate with multiple caches, which can have different eviction algorithms or sizes cache_stat_t *simulate_with_multi_caches(reader_t *reader, @@ -170,7 +173,9 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, reader_t *warmup_reader, double warmup_frac, int warmup_sec, - int num_of_threads) + int num_of_threads, + bool free_cache_when_finish, + bool use_random_seed); ``` `simulate_at_multi_sizes` allows you to pass in an array of `cache_sizes` to simulate; @@ -178,18 +183,25 @@ cache_stat_t *simulate_with_multi_caches(reader_t *reader, cache sizes `step_size, step_size*2, step_size*3 .. cache->cache_size`. `simulate_with_multi_caches` allows you to pass in an array of `cache_t` to simulate, which can have different eviction algorithms or sizes. -The return result is an array of simulation results, the users are responsible for free the array. +The return result is an array of simulation results, one per simulation, and the caller is responsible for freeing the array. ```c typedef struct { - uint64_t req_cnt; - uint64_t req_bytes; - uint64_t miss_cnt; - uint64_t miss_bytes; - uint64_t cache_size; - cache_stat_t cache_state; - void *other_data; /* not used */ -} sim_res_t; + int64_t n_warmup_req; + int64_t n_req; + int64_t n_req_byte; + double n_req_cost; + int64_t n_miss; + int64_t n_miss_byte; + double n_miss_cost; + + int64_t n_obj; + int64_t occupied_byte; + int64_t cache_size; + float sampler_ratio; + /* ... see libCacheSim/include/libCacheSim/simulator.h */ +} cache_stat_t; ``` +Object miss ratio is `n_miss / n_req` and byte miss ratio is `n_miss_byte / n_req_byte`. ### Trace utils @@ -213,10 +225,10 @@ int32_t *get_access_dist(reader_t *reader, ``` ## Examples -#### C example +### C example -#### C++ example +### C++ example ### Build a cache hierarchy with multiple layers @@ -226,16 +238,12 @@ int32_t *get_access_dist(reader_t *reader, ## FAQ -#### Linking with libCacheSim +### Linking with libCacheSim linking can be done in cmake or use pkg-config Such as in the `_build` directory: ``` export PKG_CONFIG_PATH=$PWD ``` -#### Possible problems +### Possible problems * if you get `error while loading shared libraries`, run `sudo ldconfig` - - - ---- diff --git a/doc/advanced_lib_extend.md b/doc/advanced_lib_extend.md index 547751671..e69f371ba 100644 --- a/doc/advanced_lib_extend.md +++ b/doc/advanced_lib_extend.md @@ -37,7 +37,7 @@ Specifically, you can following the steps: 3. Add `myCache_init()` function to [include/libCacheSim/evictionAlgo.h](/libCacheSim/include/libCacheSim/evictionAlgo.h). 4. Add mycache.c to [CMakeLists.txt](/libCacheSim/cache/CMakeLists.txt) so that it can be compiled. 5. Add command line option in [bin/cachesim/cache_init.h](/libCacheSim/bin/cachesim/cache_init.h) so that you can use `cachesim` binary. You may also want to take a look at [bin/cachesim/cli_parser.c](/libCacheSim/bin/cachesim/cli_parser.c). -6. Remember to add a test in [test/test_evictionAlgo.c](/test/test_evictionAlgo.c) and add the algorithm to this [README](README.md). +6. Remember to add a test in [test/test_evictionAlgo.c](/test/test_evictionAlgo.c) and add the algorithm to the [README](/README.md#supported-algorithms). > [!TIP] > Many eviction algorithms use a doubly linked list to maintain state, libCacheSim provides several functions in [cacheObj.h](/libCacheSim/include/libCacheSim/cacheObj.h) to manipulate list. diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 000000000..6e2ebf691 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,93 @@ +"""Sphinx configuration for the libCacheSim documentation. + +The docs are the Markdown files in this directory, rendered with MyST so the +same sources stay readable on GitHub. Build locally with: + + pip install -r doc/requirements.txt + sphinx-build -b html doc doc/_build/html +""" + +import os + +# -- Project information ----------------------------------------------------- + +project = "libCacheSim" +author = "Juncheng Yang" +copyright = "2024, libCacheSim authors" # noqa: A001 + +_version_file = os.path.join(os.path.dirname(__file__), os.pardir, "version.txt") +with open(_version_file, encoding="utf-8") as f: + release = f.read().strip() +version = release + +# -- General configuration --------------------------------------------------- + +extensions = ["myst_parser", "sphinxcontrib.mermaid"] + +source_suffix = {".md": "markdown", ".rst": "restructuredtext"} + +exclude_patterns = [ + "_build", + # Index for browsing the docs on GitHub; index.md is the Sphinx entry point. + "README.md", + # Not documentation. + "plot", + "assets", +] + +# Generate anchors for headings so cross-file "#section" links resolve. +myst_heading_anchors = 3 + +myst_enable_extensions = [ + "colon_fence", + "deflist", +] + +# Render ```mermaid fences as diagrams rather than trying to syntax-highlight +# them, which GitHub does natively. +myst_fence_as_directive = ["mermaid"] + +# -- HTML output ------------------------------------------------------------- + +html_theme = "sphinx_rtd_theme" +html_title = f"libCacheSim {release}" +html_static_path = [] + +# -- Link rewriting ---------------------------------------------------------- +# +# The Markdown sources are written to be read on GitHub, so links into the +# repository are root-absolute ("/libCacheSim/cache/eviction/LRU.c") or relative +# to the repository root ("../README.md"). Those resolve on github.com but not +# in a rendered docs site, so point them back at the repository. This runs on +# `source-read`, before MyST resolves links, otherwise MyST reports each one as +# a missing cross-reference. + +import re # noqa: E402 + +_REPO_BLOB_URL = "https://github.com/1a1a11a/libCacheSim/blob/develop" + +# Markdown inline links whose target leaves this directory. +_LINK_RE = re.compile(r"\]\((/[^)\s]*|\.\./[^)\s]*)\)") + + +def _rewrite_target(match): + target = match.group(1) + + # Pages in this build: keep them as local cross-references so the sidebar, + # search, and PDF output link them properly. + if target.startswith("/doc/"): + return "](%s)" % target[len("/doc/") :] + + if target.startswith("../"): + target = "/" + target[len("../") :] + + return "](%s%s)" % (_REPO_BLOB_URL, target) + + +def _rewrite_repo_links(app, docname, source): + source[0] = _LINK_RE.sub(_rewrite_target, source[0]) + + +def setup(app): + app.connect("source-read", _rewrite_repo_links) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 000000000..c113dca3d --- /dev/null +++ b/doc/index.md @@ -0,0 +1,51 @@ +# libCacheSim + +A high-performance library for building and running cache simulations. + +libCacheSim ships three things: + +* **cachesim**, a high-performance cache simulator for running cache simulations. +* **traceAnalyzer**, a high-performance and versatile analyzer for cache traces. +* **libCacheSim**, a library for building your own cache simulators. + +New here? Start with [Install & Build](install.md), then [the cachesim guide](quickstart_cachesim.md). + +The commands throughout these pages are run from the build directory (`_build/` if you followed the [README](https://github.com/1a1a11a/libCacheSim#build-and-install-libcachesim)), so the sample traces in `data/` are at `../data/`. + +```{toctree} +:maxdepth: 2 +:caption: Getting started + +install +quickstart_cachesim +quickstart_traceAnalyzer +quickstart_traceUtils +quickstart_mrcProfiler +quickstart_plugin +``` + +```{toctree} +:maxdepth: 2 +:caption: Using libCacheSim as a library + +advanced_lib +advanced_lib_extend +API +``` + +```{toctree} +:maxdepth: 2 +:caption: Performance and debugging + +performance +memory_usage_profiling +debug +``` + +## Other resources + +* [Python binding](https://github.com/cacheMon/libCacheSim-python) — easier API access, `pip install libcachesim` +* [FAQ](https://github.com/1a1a11a/libCacheSim/blob/develop/FAQ.md) +* [Contributing](https://github.com/1a1a11a/libCacheSim/blob/develop/CONTRIBUTING.md) +* [Open-source cache datasets](https://github.com/cacheMon/cache_dataset) +* [Issue tracker](https://github.com/1a1a11a/libCacheSim/issues) and [Discussions](https://github.com/1a1a11a/libCacheSim/discussions) diff --git a/doc/install.md b/doc/install.md index ec91f79b5..f0270c4e6 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,19 +1,19 @@ -## Install dependency +# Install dependency libCacheSim uses [cmake](https://cmake.org/) build system with [Ninja](https://ninja-build.org/) generator and has a few dependencies: [glib](https://developer.gnome.org/glib/) [tcmalloc](https://github.com/google/tcmalloc), [zstd](https://github.com/facebook/zstd). -### Install dependency on Ubuntu +## Install dependency on Ubuntu -#### Install glib, tcmalloc, cmake and ninja +### Install glib, tcmalloc, cmake and ninja ```bash sudo apt install libglib2.0-dev libgoogle-perftools-dev cmake ninja-build ``` -#### Install zstd +### Install zstd zstd must be installed from source @@ -27,7 +27,7 @@ sudo ninja install popd ``` -#### Install XGBoost [Optional] +### Install XGBoost [Optional] ```bash git clone --recursive https://github.com/dmlc/xgboost @@ -38,7 +38,7 @@ sudo ninja install popd ``` -#### Install LightGBM [Optional] +### Install LightGBM [Optional] ```bash git clone --recursive https://github.com/microsoft/LightGBM @@ -49,7 +49,7 @@ sudo ninja install popd ``` -### Install dependency on Mac +## Install dependency on Mac using [homebrew](https://brew.sh/) as an example. While the first line is necessary, the following two lines needs to be run if you encounter errors including: @@ -62,7 +62,7 @@ brew install argp-standalone brew install pkg-config ``` -#### Install zstd +### Install zstd Use the below command to install ```bash brew install zstd diff --git a/doc/memory_usage_profiling.md b/doc/memory_usage_profiling.md index 024e5c0ef..695fa22d3 100644 --- a/doc/memory_usage_profiling.md +++ b/doc/memory_usage_profiling.md @@ -36,7 +36,7 @@ ms_print ./massif.out > massif.result The generated report primarily includes a bar chart of memory usage (with instructions executed as the x-axis) and several heap profile snapshots. Some snapshots display detailed function call relationships showing how memory was allocated. Below is an example of such a report: -```sh +```text MB 519.4^ : |#:::::::::::::@::::::::::::::::@::@@::::::::::::::::::::::::@::@::@::@:: diff --git a/doc/performance.md b/doc/performance.md index 9ae42cc40..d38983487 100644 --- a/doc/performance.md +++ b/doc/performance.md @@ -1,15 +1,66 @@ +# Performance tuning +libCacheSim is built for high-throughput trace replay. This page collects the knobs that matter and how to measure their effect on your own workload — the numbers depend heavily on the trace, the algorithm, and the machine, so measure rather than assume. -## Performance +## Measuring throughput +`cachesim` reports throughput (in millions of requests per second) on every run: +```bash +cd _build +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 1gb +``` -## Memory efficiency +For a systematic comparison across algorithms and working set sizes, [`scripts/benchmark_throughput.py`](/scripts/benchmark_throughput.py) generates Zipfian traces and sweeps them: +```bash +cd scripts +python3 benchmark_throughput.py --help +``` +The sample traces in [`data/`](/data/) are far too small to benchmark with — they fit in cache and are dominated by startup cost. Use a trace of at least a few million requests. -### Other -#### Performance Optimizations -* hugepage - to turn on hugepage support, please do `echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled` +## Build configuration +Build in release mode. A debug build is several times slower, and [`scripts/debug.sh`](/scripts/debug.sh) additionally skips tcmalloc to stay debugger-friendly: +```bash +cmake -G Ninja -B _build -DCMAKE_BUILD_TYPE=Release +``` + +**tcmalloc** is linked automatically when CMake finds it, and matters because the hot path allocates per-object metadata. `cmake` prints `!!! cannot find tcmalloc` when it is missing; install it (`libgoogle-perftools-dev` on Debian/Ubuntu, `gperftools` via Homebrew) and reconfigure. + +**Transparent hugepages** are enabled at compile time by default (`USE_HUGEPAGE=ON`), which reduces TLB misses on the hash table. They also need to be enabled on the host: + +```bash +echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled +``` + +Turn the compile-time option off with `-DUSE_HUGEPAGE=OFF` if your environment does not support them. + +## Trace format + +Binary formats are several times faster than csv, because csv parsing dominates replay for fast algorithms. Convert once with `traceConv` and reuse: + +```bash +./bin/traceConv ../data/cloudPhysicsIO.csv csv \ + -t "time-col=2,obj-id-col=5,obj-size-col=4,obj-id-is-num=1" \ + --output-format=oracleGeneral +``` + +See [quickstart_traceUtils.md](quickstart_traceUtils.md). zstd-compressed binary traces are read without decompressing first, so compression costs little replay time while saving substantial disk. + +When the id column holds numbers, pass `obj-id-is-num=true` so the reader skips hashing. + +## Runtime options + +* `--num-thread=N` — simulations across algorithms and cache sizes are run in parallel, so a sweep costs little more than its slowest single run. +* `--ignore-obj-size 1` — treats every object as size one. Faster, and the right choice when you want object miss ratio rather than byte miss ratio. +* `--consider-obj-metadata=false` — skips accounting for per-algorithm metadata overhead in the cache size. +* `--num-req=N` — caps how much of the trace is read, useful when iterating. + +## Memory + +Memory is dominated by the hash table and per-object metadata, so it scales with the number of *objects* rather than the number of requests. `--ignore-obj-size 1` with a small cache size keeps the object count down. + +To profile actual usage, see [memory_usage_profiling.md](memory_usage_profiling.md). diff --git a/doc/quickstart_traceAnalyzer.md b/doc/quickstart_traceAnalyzer.md index 556223153..8c89ab19e 100644 --- a/doc/quickstart_traceAnalyzer.md +++ b/doc/quickstart_traceAnalyzer.md @@ -1,17 +1,17 @@ -## Trace analysis tool +# Trace analysis tool libCacheSim provides a set of tools to help you analyze traces. After building the project, you can find a binary called `traceAnalyzer`. This doc shows how to use the tool. If you are interested, the source code is located in the [bin/traceAnalyzer/](/libCacheSim/bin/traceAnalyzer) and [traceAnalyzer](/libCacheSim/traceAnalyzer) directory. -### Obtain trace statistics -#### Usage: +## Obtain trace statistics +### Usage: ``` # ./bin/traceAnalyzer --help for a list of tasks and options ./bin/traceAnalyzer PATH_TO_TRACE traceType [--task1] [--task2] ``` -#### A list of tasks: +### A list of tasks: * `--common`: run all common tasks, including `--stat`, `--traceStat`, `--reqRate`, `--size`, `--reuse`, `--popularity` * `--all`: run all tasks * `--accessPattern`: generate access pattern data for plotting using [scripts/traceAnalysis/access_pattern.py](/scripts/traceAnalysis/access_pattern.py) @@ -21,7 +21,7 @@ If you are interested, the source code is located in the [bin/traceAnalyzer/](/l * `--popularity`: generate popularity data for plotting using [scripts/traceAnalysis/popularity.py](/scripts/traceAnalysis/popularity.py) * `--popularityDecay`: generate popularity data for plotting using [scripts/traceAnalysis/popularity_decay.py](/scripts/traceAnalysis/popularity_decay.py) -#### Example: +### Example: ```bash # run all common tasks ./bin/traceAnalyzer PATH_TO_TRACE traceType --common @@ -69,11 +69,11 @@ The trace analyzer will generate statistics of the trace and save them to `stat` ---- -### Plot trace statistics and visualize the trace +## Plot trace statistics and visualize the trace We provide plot scripts in [scripts/traceAnalysis/](/scripts/traceAnalysis/) to help you plot the trace statistics. After generating plot data, we can plot access pattern, request rate, size, reuse, and popularity using the following commands: -#### Access pattern +### Access pattern ```bash # plot the access pattern using wall clock (real) time python3 scripts/traceAnalysis/access_pattern.py ${dataname}.accessRtime @@ -105,7 +105,7 @@ The first 10m requests of the Twitter cluster52 trace, this is a Zipf workload. -#### Request rate +### Request rate ```bash # this is only supported for traces that have (wall clock) time field python3 scripts/traceAnalysis/req_rate.py ${dataname}.reqRate_w300 @@ -124,7 +124,7 @@ The block workload has a daily request spike, while the Twitter workload is too
-#### Size distribution +### Size distribution ```bash # this is only supported for traces that have object size python3 scripts/traceAnalysis/size.py ${dataname}.size @@ -144,7 +144,7 @@ The Request curve is weighted by request count, and the Object curve is weighted
-#### Reuse distribution +### Reuse distribution This is the time since the last access of the object. ```bash @@ -173,7 +173,7 @@ The first 10m requests of the Twitter cluster52 trace. The left column shows wal
-#### Popularity +### Popularity ```bash # the popularity skewness ($\alpha$) is in the output of traceAnalyzer # this plots the request count/freq over object rank @@ -195,7 +195,7 @@ The first 10m requests of the Twitter cluster52 trace.
-#### Size distribution heatmap +### Size distribution heatmap This and the following plots are more expensive plots that require more CPU cycles and DRAM usage to generate. This plot requires wall clock time and object size in the trace. This is a heatmap of the size distribution of the trace. The x-axis is the clock time, and the y-axis is the size. The color represents the number of requests having a certain size range at that time. The darker the color, the more requests of the certain size at that time. @@ -217,7 +217,7 @@ Left: a block cache workload (w92), right: the first 10m requests of the Twitter
-#### Reuse distribution heatmap +### Reuse distribution heatmap This is a heatmap of the reuse distribution of the trace. The x-axis is the wall clock time, and the y-axis is the reuse time (in seconds) or reuse distance (the number of requests since last access of the object). The color represents the number of requests having the reuse time or reuse distance. The heatmap is generated using the following command: @@ -237,7 +237,7 @@ Left: a block cache workload (w92), right: the first 10m requests of the Twitter
-#### popularity decay +### popularity decay There are two versions of the plots, one is line plot, and the other is a heatmap. ```bash @@ -257,7 +257,7 @@ The Request curve is weighted by request count, and the Object curve is weighted
--> -### Advanced features +## Advanced features ```bash # cap the number of requests read from the trace ./bin/traceAnalyzer --num-req=1000000 ../data/cloudPhysicsIO.vscsi vscsi diff --git a/doc/quickstart_traceUtils.md b/doc/quickstart_traceUtils.md index 78fc8db2a..cd31fba04 100644 --- a/doc/quickstart_traceUtils.md +++ b/doc/quickstart_traceUtils.md @@ -1,7 +1,7 @@ -## Other trace utilities +# Other trace utilities We also provide some trace utilities to help you use the traces and debug applications. -### tracePrint +## tracePrint Print requests from a trace. ```bash @@ -9,7 +9,7 @@ Print requests from a trace. ./bin/tracePrint ../data/cloudPhysicsIO.vscsi vscsi -n 10 ``` -### traceConv +## traceConv Convert a trace to oracleGeneral format so you can run it faster (10x speedup) using less memory. Meanwhile, the generated trace has a smaller size, contains next request time. ```bash # the first parameter is the input trace, the second parameter is trace type, the output is in the same directory with suffic oracleGeneral @@ -28,7 +28,7 @@ We can also sample a trace to reduce its size. ./bin/traceConv ../data/cloudPhysicsIO.vscsi vscsi -s 0.01 --output-format=oracleGeneral ``` -### traceFilter +## traceFilter traceFilter simulates a multi-layer cache hierarchy. It filters the trace based on the cache hit/miss information and generates a trace for the second layer. The generated trace is in oracleGeneral format. diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 000000000..025754ded --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,6 @@ +# Documentation build only. The plotting/analysis scripts use the root +# requirements.txt instead. +sphinx>=7.0 +myst-parser>=2.0 +sphinx-rtd-theme>=2.0 +sphinxcontrib-mermaid>=0.9 From 5d007cbbecd300d821c8f46713fe6db56356ef68 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 00:55:11 +0000 Subject: [PATCH 09/33] test: cover the CLI paths that were crashing Nothing exercised the command-line tools; every crash fixed on this branch was reachable from a documented command and none of them would have been caught. testCLI runs the built binaries and asserts on exit status and output. Covered: - each sample trace format replays, and a numeric csv id column without obj-id-is-num is rejected rather than silently hashed - `-e print` for SLRU, QDLP, S3FIFOd and ten other algorithms, checking the reported values, since these dereferenced state that parse_params runs before - SLRU parameter validation: n-seg of 0, -1 and above the maximum, empty seg-size, seg-size summing to zero, and more segments than the array holds - value-taking options in the space-separated form (`-o path`), which argp passes as a NULL argument when the option is declared OPTION_ARG_OPTIONAL, plus the attached and `=` forms, and bare flags like --verbose that reach is_true(NULL) Invalid input is asserted to fail *cleanly*: a non-zero exit with a message, but not SIGSEGV, SIGFPE or SIGBUS. The project's ERROR() aborts, so a deliberate rejection is distinguishable from a crash. Confirmed the tests actually catch these: reverting the six fixed sources to develop and rebuilding turns up 16 failures, and 0 with them restored. The script skips itself when the binaries or sample traces are missing, so it stays green in build layouts it does not understand. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- CONTRIBUTING.md | 6 ++ test/CMakeLists.txt | 7 ++ test/test_cli.sh | 220 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100755 test/test_cli.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4db3c3bc..0173db26c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,6 +53,12 @@ CI additionally builds Ubuntu with LeakSanitizer, so please check that new alloc Every new eviction, admission, or prefetching algorithm needs a test in the matching file under [`test/`](test/) — for eviction algorithms that is [`test/test_evictionAlgo.c`](test/test_evictionAlgo.c). +[`test/test_cli.sh`](test/test_cli.sh) (the `testCLI` target) covers the command-line tools rather than the library: option parsing, `-e print` parameter reporting, and eviction parameter validation. If you add an algorithm parameter or a CLI option, add a case there. It runs from the build directory and skips itself if the binaries or the sample traces are not where it expects, so it can also be run by hand: + +```bash +cd _build && bash ../test/test_cli.sh +``` + ## Code style * The project follows **Google style**: 2-space indent, 80-column limit, configured in [`.clang-format`](.clang-format) and [`.clang-tidy`](.clang-tidy). Run `clang-format -i ` before committing. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a3ce4c3f3..b40fd5e0a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -59,6 +59,13 @@ add_test(NAME testAdmissionAlgo COMMAND testAdmissionAlgo WORKING_DIRECTORY .) add_test(NAME testPrefetchAlgo COMMAND testPrefetchAlgo WORKING_DIRECTORY .) add_test(NAME testDataStructure COMMAND testDataStructure WORKING_DIRECTORY .) add_test(NAME testUtils COMMAND testUtils WORKING_DIRECTORY .) + +# Exercises the command-line tools themselves: argument handling, `-e print` +# parameter reporting, and eviction parameter validation. Runs from the build +# directory so it can find bin/ and ../data/; skips itself if either is absent. +add_test(NAME testCLI + COMMAND ${CMAKE_COMMAND} -E env bash ${CMAKE_CURRENT_SOURCE_DIR}/test_cli.sh + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) # add_test(NAME testMrcProfiler COMMAND testMrcProfiler WORKING_DIRECTORY .) # if (ENABLE_GLCACHE) diff --git a/test/test_cli.sh b/test/test_cli.sh new file mode 100755 index 000000000..c27aea2ce --- /dev/null +++ b/test/test_cli.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# +# Regression tests for the command-line tools. +# +# These cover argument handling and parameter reporting, which is not exercised +# by the C unit tests. Every case here crashed at some point: `-e print` +# dereferenced state that had not been built yet, options declared +# OPTION_ARG_OPTIONAL were handed a NULL argument in the space-separated form, +# and SLRU never validated n-seg. + +set -uo pipefail + +# Locate the binaries and the sample traces. ctest runs this from the build +# directory, but allow running it by hand from elsewhere too. +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +BIN_DIR="" +for candidate in "./bin" "../bin" "${SCRIPT_DIR}/../_build/bin" "${SCRIPT_DIR}/../build/bin"; do + if [[ -x "${candidate}/cachesim" ]]; then + BIN_DIR=$(cd "${candidate}" && pwd) + break + fi +done +if [[ -z "${BIN_DIR}" ]]; then + echo "SKIP: cannot find the built binaries (looked for bin/cachesim)" + exit 0 +fi + +DATA_DIR="" +for candidate in "./data" "../data" "../../data" "${SCRIPT_DIR}/../data"; do + if [[ -f "${candidate}/cloudPhysicsIO.vscsi" ]]; then + DATA_DIR=$(cd "${candidate}" && pwd) + break + fi +done +if [[ -z "${DATA_DIR}" ]]; then + echo "SKIP: cannot find the sample traces (looked for data/cloudPhysicsIO.vscsi)" + exit 0 +fi + +TRACE="${DATA_DIR}/cloudPhysicsIO.vscsi" +TRACE_ORACLE="${DATA_DIR}/cloudPhysicsIO.oracleGeneral.bin" +TRACE_CSV="${DATA_DIR}/cloudPhysicsIO.csv" +TRACE_TXT="${DATA_DIR}/cloudPhysicsIO.txt" + +WORK_DIR=$(mktemp -d) +trap 'rm -rf "${WORK_DIR}"' EXIT +cd "${WORK_DIR}" || exit 1 + +N_PASS=0 +N_FAIL=0 + +# Signals that mean a crash rather than a reported error. The project's ERROR() +# aborts (134), which is a deliberate exit, not a crash. +SIGSEGV_RC=139 +SIGFPE_RC=136 +SIGBUS_RC=138 + +_report() { + if [[ $1 -eq 0 ]]; then + N_PASS=$((N_PASS + 1)) + else + N_FAIL=$((N_FAIL + 1)) + echo " FAIL: $2" + fi +} + +# Command must exit 0. +expect_ok() { + local desc=$1 + shift + local out + out=$("$@" 2>&1) + local rc=$? + if [[ ${rc} -eq 0 ]]; then + _report 0 "" + else + _report 1 "${desc} (exit ${rc})" + echo "${out}" | tail -3 | sed 's/^/ /' + fi +} + +# Command must exit 0 and its output must match a pattern. +expect_output() { + local desc=$1 pattern=$2 + shift 2 + local out + out=$("$@" 2>&1) + local rc=$? + if [[ ${rc} -ne 0 ]]; then + _report 1 "${desc} (exit ${rc})" + echo "${out}" | tail -3 | sed 's/^/ /' + elif ! grep -qE "${pattern}" <<<"${out}"; then + _report 1 "${desc} (output did not match /${pattern}/)" + echo "${out}" | tail -3 | sed 's/^/ /' + else + _report 0 "" + fi +} + +# Invalid input must be rejected with a message, not a crash. +expect_clean_error() { + local desc=$1 + shift + local out + out=$("$@" 2>&1) + local rc=$? + if [[ ${rc} -eq ${SIGSEGV_RC} || ${rc} -eq ${SIGFPE_RC} || ${rc} -eq ${SIGBUS_RC} ]]; then + _report 1 "${desc} crashed with signal (exit ${rc})" + elif [[ ${rc} -eq 0 ]]; then + _report 1 "${desc} was accepted but should have been rejected" + elif ! grep -qi "error" <<<"${out}"; then + _report 1 "${desc} failed without an error message (exit ${rc})" + else + _report 0 "" + fi +} + +echo "running cachesim tests" + +# Each supported sample trace format replays. +expect_output "cachesim vscsi" "miss ratio" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi lru 1gb +expect_output "cachesim txt" "miss ratio" \ + "${BIN_DIR}/cachesim" "${TRACE_TXT}" txt lru 1gb +expect_output "cachesim oracleGeneral" "miss ratio" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral lru 1gb +expect_output "cachesim csv" "miss ratio" \ + "${BIN_DIR}/cachesim" "${TRACE_CSV}" csv lru 1gb \ + -t "time-col=2, obj-id-col=5, obj-size-col=4, obj-id-is-num=true" + +# A numeric id column without obj-id-is-num is rejected, not silently hashed. +expect_clean_error "cachesim csv without obj-id-is-num" \ + "${BIN_DIR}/cachesim" "${TRACE_CSV}" csv lru 1gb \ + -t "time-col=2, obj-id-col=5, obj-size-col=4" + +echo "running -e print tests" + +# `-e print` runs before the cache is fully built, so the reporting path must +# not touch anything that is still uninitialized. +expect_output "slru -e print" "n-seg=4,seg-size=25:25:25:25" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e print +expect_output "slru -e n-seg=8,print" "n-seg=8" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "n-seg=8,print" +expect_output "slru -e seg-size=1:2:3:4,print" "seg-size=9:19:29:39" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "seg-size=1:2:3:4,print" +expect_output "slru -e print with auto sizing" "n-seg=" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru auto -e print +expect_output "qdlp -e print" "fifo-size-ratio=" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral qdlp 1gb -e print +expect_output "s3fifod -e print" "fifo-size-ratio=" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral s3fifod 1gb -e print + +for algo in arc car clock clockpro lecar lru-prob beladysize fifo-merge s3fifo 2q; do + expect_ok "${algo} -e print" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 1gb -e print +done + +echo "running SLRU parameter validation tests" + +# n-seg divides the cache size and the reported percentages, and seg-size fills +# a fixed-size array, so both are bounded. +expect_clean_error "slru n-seg=0" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "n-seg=0" +expect_clean_error "slru n-seg=0 with print" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "n-seg=0,print" +expect_clean_error "slru n-seg=-1" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "n-seg=-1" +expect_clean_error "slru n-seg above the maximum" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "n-seg=64" +expect_clean_error "slru empty seg-size" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "seg-size=,print" +expect_clean_error "slru seg-size sums to zero" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "seg-size=0:0" +expect_clean_error "slru too many segments" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb \ + -e "seg-size=1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1" +expect_clean_error "slru unknown parameter" \ + "${BIN_DIR}/cachesim" "${TRACE}" vscsi slru 1gb -e "no-such-param=1" + +echo "running option parsing tests" + +# Options that take a value must accept the space-separated form. Declaring +# them OPTION_ARG_OPTIONAL made argp pass a NULL argument for "-o path". +if [[ -x "${BIN_DIR}/traceAnalyzer" ]]; then + expect_ok "traceAnalyzer -o PATH" \ + "${BIN_DIR}/traceAnalyzer" -o out-spaced "${TRACE}" vscsi + expect_ok "traceAnalyzer -oPATH" \ + "${BIN_DIR}/traceAnalyzer" -oout-attached "${TRACE}" vscsi + expect_ok "traceAnalyzer --output=PATH" \ + "${BIN_DIR}/traceAnalyzer" --output=out-equals "${TRACE}" vscsi + # A bare OPTION_ARG_OPTIONAL flag passes a NULL argument through is_true(). + expect_ok "traceAnalyzer --verbose" \ + "${BIN_DIR}/traceAnalyzer" --verbose "${TRACE}" vscsi + expect_ok "traceAnalyzer --common" \ + "${BIN_DIR}/traceAnalyzer" --common -o out-common "${TRACE}" vscsi + expect_ok "traceAnalyzer --num-req" \ + "${BIN_DIR}/traceAnalyzer" --num-req=10000 -o out-nreq "${TRACE}" vscsi +fi + +if [[ -x "${BIN_DIR}/mrcProfiler" ]]; then + expect_ok "mrcProfiler with space-separated options" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo LRU --profiler SHARDS --profiler-params FIX_RATE,0.01,42 --size 0.1,0.5,10 + expect_ok "mrcProfiler with = options" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 + expect_ok "mrcProfiler -o PATH" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 \ + -o mrc-out + expect_ok "mrcProfiler --ignore-obj-size" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 \ + --ignore-obj-size +fi + +echo +echo "${N_PASS} passed, ${N_FAIL} failed" +[[ ${N_FAIL} -eq 0 ]] From 5b5d98be997b70f573280bc53568583ad53660ae Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 01:22:16 +0000 Subject: [PATCH 10/33] fix: leaks and uninitialized reads found by the new CLI test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ubuntu job builds with LeakSanitizer, and testCLI runs the binaries to completion, so it started reporting what the C unit tests never reached. All of these are pre-existing; none are caused by this branch. Leaked on the `-e print` early exit (29 files, one line each): the parameter string is strdup'd, but the print branch exits before the free at the end of the parser. GLCache, Mithril and PG never kept the original pointer at all, so they leaked on every call rather than only on print. Missing frees, which only show up when a cache is actually torn down: - Size_free freed the priority queue and its nodes but not the params struct itself; RandomLRU_free and SLRUv0_free likewise - S3FIFOd_free freed three of its five sub-caches, leaving small_eviction and main_eviction (73 KB on a short replay) - SLRUv0_cool allocated a request up front and returned early on i == 0 without freeing it, once per eviction from the bottom segment - WTinyLFU_free never freed its params struct Two more crashes of the kind this branch already fixed elsewhere, both reachable from documented commands: - `s3fifov0 -e print` and `flashProb -e print` dereferenced a sub-cache that parse_params runs before building - WTinyLFU_init read params->main_cache->obj_md_size out of a malloc'd, un-memset struct before main_cache was assigned, so `wtinyLFU --consider-obj-metadata=true` segfaulted. The struct is now zeroed and the metadata size read after the sub-cache exists. `-e print` is documented for every algorithm but Clock2QPlus and pluginCache rejected the bare `print` key before reaching their own print branch, and WTinyLFU had no print branch at all despite taking parameters. All three now behave like the rest. Reverted the change to cachesim's hashpower heuristic. It tests for "data/trace." and has been dead for as long as that file has not existed; pointing it at the current sample traces revives it, and a hash table 256 times smaller changes which candidates sampling-based algorithms draw — RandomLRU's miss ratio moved in the fourth decimal. Left dead with a note rather than silently changing results. Verified against a develop build: identical miss ratios for 17 algorithms, and `wtinyLFU --consider-obj-metadata=true` goes from SIGSEGV to a result. ctest is 10/10 both plain and under LSan. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/bin/cachesim/cache_init.h | 11 +++- libCacheSim/cache/eviction/ARC.c | 1 + libCacheSim/cache/eviction/ARCv0.c | 1 + libCacheSim/cache/eviction/BeladySize.c | 1 + libCacheSim/cache/eviction/CAR.c | 1 + libCacheSim/cache/eviction/Clock.c | 1 + libCacheSim/cache/eviction/Clock2QPlus.c | 4 +- libCacheSim/cache/eviction/ClockPro.c | 1 + libCacheSim/cache/eviction/FIFO_Merge.c | 1 + libCacheSim/cache/eviction/FIFO_Reinsertion.c | 1 + libCacheSim/cache/eviction/GLCache/GLCache.c | 4 ++ libCacheSim/cache/eviction/Hyperbolic.c | 1 + libCacheSim/cache/eviction/LRUProb.c | 1 + libCacheSim/cache/eviction/LeCaR.c | 1 + libCacheSim/cache/eviction/QDLP.c | 1 + libCacheSim/cache/eviction/RandomLRU.c | 2 + libCacheSim/cache/eviction/S3FIFO.c | 1 + libCacheSim/cache/eviction/S3FIFOd.c | 4 ++ libCacheSim/cache/eviction/S3FIFOv0.c | 7 +- libCacheSim/cache/eviction/SLRU.c | 1 + libCacheSim/cache/eviction/SLRUv0.c | 7 +- libCacheSim/cache/eviction/Size.c | 1 + libCacheSim/cache/eviction/TwoQ.c | 1 + libCacheSim/cache/eviction/WTinyLFU.c | 34 ++++++++-- libCacheSim/cache/eviction/cpp/LRU_K.cpp | 1 + libCacheSim/cache/eviction/fifo/LP_ARC.c | 1 + libCacheSim/cache/eviction/fifo/LP_SFIFO.c | 1 + libCacheSim/cache/eviction/fifo/LP_TwoQ.c | 1 + libCacheSim/cache/eviction/fifo/SFIFO.c | 1 + libCacheSim/cache/eviction/fifo/SFIFOv0.c | 1 + libCacheSim/cache/eviction/other/S3LRU.c | 1 + libCacheSim/cache/eviction/other/flashProb.c | 12 ++-- libCacheSim/cache/eviction/plugin_cache.c | 5 +- libCacheSim/cache/prefetch/Mithril.c | 4 ++ libCacheSim/cache/prefetch/PG.c | 4 ++ test/test_cli.sh | 66 ++++++++++++++++++- 36 files changed, 165 insertions(+), 22 deletions(-) diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index 34d85aae1..bb7c243cd 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -27,9 +27,14 @@ static inline cache_t *create_cache(const char *trace_path, }; cache_t *cache; - /* the trace provided is small */ - if (trace_path != NULL && strstr(trace_path, "data/trace.") != NULL) - cc_params.hashpower -= 8; + /* NOTE: there used to be a heuristic here shrinking hashpower by 8 when the + * trace path contained "data/trace.", to save memory on the sample traces. + * No such file has existed for a long time, so it never fired. Pointing it at + * the current sample traces is not a free fix: a smaller hash table changes + * which candidates sampling-based algorithms (RandomLRU, Hyperbolic, ...) + * draw, so miss ratios shift. Left out rather than silently changing results; + * re-add deliberately if the memory saving is worth that. */ + typedef struct { const char *name; cache_t *(*init_func)(common_cache_params_t, const char *); diff --git a/libCacheSim/cache/eviction/ARC.c b/libCacheSim/cache/eviction/ARC.c index c8b22d88e..2b25147f4 100644 --- a/libCacheSim/cache/eviction/ARC.c +++ b/libCacheSim/cache/eviction/ARC.c @@ -627,6 +627,7 @@ static void ARC_parse_params(cache_t *cache, if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", ARC_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/ARCv0.c b/libCacheSim/cache/eviction/ARCv0.c index 98e8c6831..e4613062b 100644 --- a/libCacheSim/cache/eviction/ARCv0.c +++ b/libCacheSim/cache/eviction/ARCv0.c @@ -544,6 +544,7 @@ static void ARCv0_parse_params(cache_t *cache, if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", ARCv0_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/BeladySize.c b/libCacheSim/cache/eviction/BeladySize.c index 78f6d0236..d0ebb00fa 100644 --- a/libCacheSim/cache/eviction/BeladySize.c +++ b/libCacheSim/cache/eviction/BeladySize.c @@ -320,6 +320,7 @@ static void BeladySize_parse_params(cache_t *cache, } } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", BeladySize_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s, support %s\n", cache->cache_name, diff --git a/libCacheSim/cache/eviction/CAR.c b/libCacheSim/cache/eviction/CAR.c index 74eaea502..8b4c23da7 100644 --- a/libCacheSim/cache/eviction/CAR.c +++ b/libCacheSim/cache/eviction/CAR.c @@ -488,6 +488,7 @@ static void CAR_parse_params(cache_t *cache, } } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", CAR_current_params(cache, params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s, example parameters %s\n", diff --git a/libCacheSim/cache/eviction/Clock.c b/libCacheSim/cache/eviction/Clock.c index d6c6a4003..d13190bfa 100644 --- a/libCacheSim/cache/eviction/Clock.c +++ b/libCacheSim/cache/eviction/Clock.c @@ -328,6 +328,7 @@ static void Clock_parse_params(cache_t *cache, } } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", Clock_current_params(cache, params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s, example parameters %s\n", diff --git a/libCacheSim/cache/eviction/Clock2QPlus.c b/libCacheSim/cache/eviction/Clock2QPlus.c index 7b6bb4658..373664fcf 100644 --- a/libCacheSim/cache/eviction/Clock2QPlus.c +++ b/libCacheSim/cache/eviction/Clock2QPlus.c @@ -491,7 +491,8 @@ static void Clock2QPlus_parse_params(cache_t *cache, params_str++; } - if (key == NULL || value == NULL) { + /* "print" is a bare flag, not a key=value pair */ + if (key == NULL || (value == NULL && strcasecmp(key, "print") != 0)) { ERROR("invalid parameter string: missing key or value\n"); exit(1); } @@ -506,6 +507,7 @@ static void Clock2QPlus_parse_params(cache_t *cache, params->move_to_main_threshold = atoi(value); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", Clock2QPlus_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/ClockPro.c b/libCacheSim/cache/eviction/ClockPro.c index c1fa84273..6e070c764 100644 --- a/libCacheSim/cache/eviction/ClockPro.c +++ b/libCacheSim/cache/eviction/ClockPro.c @@ -514,6 +514,7 @@ static void ClockPro_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", ClockPro_current_params(cache, params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/FIFO_Merge.c b/libCacheSim/cache/eviction/FIFO_Merge.c index 763ab61f3..afc358635 100644 --- a/libCacheSim/cache/eviction/FIFO_Merge.c +++ b/libCacheSim/cache/eviction/FIFO_Merge.c @@ -386,6 +386,7 @@ static void FIFO_Merge_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("%s parameters: %s\n", cache->cache_name, FIFO_Merge_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/FIFO_Reinsertion.c b/libCacheSim/cache/eviction/FIFO_Reinsertion.c index 3d68a7848..279ae3bd6 100644 --- a/libCacheSim/cache/eviction/FIFO_Reinsertion.c +++ b/libCacheSim/cache/eviction/FIFO_Reinsertion.c @@ -409,6 +409,7 @@ static void FIFO_Reinsertion_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("%s parameters: %s\n", cache->cache_name, FIFO_Reinsertion_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/GLCache/GLCache.c b/libCacheSim/cache/eviction/GLCache/GLCache.c index 12ef0c56b..ba788204b 100644 --- a/libCacheSim/cache/eviction/GLCache/GLCache.c +++ b/libCacheSim/cache/eviction/GLCache/GLCache.c @@ -59,6 +59,7 @@ const char *GLCache_default_params(void) { static void GLCache_parse_init_params(const char *cache_specific_params, GLCache_params_t *params) { char *params_str = strdup(cache_specific_params); + char *old_params_str = params_str; while (params_str != NULL && params_str[0] != '\0') { char *key = strsep((char **)¶ms_str, "="); @@ -104,13 +105,16 @@ static void GLCache_parse_init_params(const char *cache_specific_params, } else if (strcasecmp(key, "print") == 0 || strcasecmp(key, "default") == 0) { printf("default params: %s\n", GLCache_default_params()); + free(old_params_str); exit(0); } else { ERROR("GLCache does not have parameter %s\n", key); printf("default params: %s\n", GLCache_default_params()); + free(old_params_str); exit(1); } } + free(old_params_str); } // *********************************************************************** diff --git a/libCacheSim/cache/eviction/Hyperbolic.c b/libCacheSim/cache/eviction/Hyperbolic.c index fb4badc80..462e6eddb 100644 --- a/libCacheSim/cache/eviction/Hyperbolic.c +++ b/libCacheSim/cache/eviction/Hyperbolic.c @@ -276,6 +276,7 @@ static void Hyperbolic_parse_params(cache_t *cache, } } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", Hyperbolic_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s, support %s\n", cache->cache_name, diff --git a/libCacheSim/cache/eviction/LRUProb.c b/libCacheSim/cache/eviction/LRUProb.c index 23ada9e8c..1bc32bc40 100644 --- a/libCacheSim/cache/eviction/LRUProb.c +++ b/libCacheSim/cache/eviction/LRUProb.c @@ -280,6 +280,7 @@ static void LRU_Prob_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", LRU_Prob_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/LeCaR.c b/libCacheSim/cache/eviction/LeCaR.c index eacf53358..228cd1dd7 100644 --- a/libCacheSim/cache/eviction/LeCaR.c +++ b/libCacheSim/cache/eviction/LeCaR.c @@ -621,6 +621,7 @@ static void LeCaR_parse_params(cache_t *cache, params->w_lru = (double)strtod(value, &end); } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", LeCaR_current_params(cache, params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/QDLP.c b/libCacheSim/cache/eviction/QDLP.c index a1eb19890..d42b12ab5 100644 --- a/libCacheSim/cache/eviction/QDLP.c +++ b/libCacheSim/cache/eviction/QDLP.c @@ -474,6 +474,7 @@ static void QDLP_parse_params(cache_t *cache, strncpy(params->main_cache_type, value, 30); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", QDLP_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/RandomLRU.c b/libCacheSim/cache/eviction/RandomLRU.c index 450157d24..8293e7b74 100644 --- a/libCacheSim/cache/eviction/RandomLRU.c +++ b/libCacheSim/cache/eviction/RandomLRU.c @@ -98,6 +98,7 @@ cache_t *RandomLRU_init(const common_cache_params_t ccache_params, static void RandomLRU_free(cache_t *cache) { RandomLRU_params_t *params = (RandomLRU_params_t *)(cache->eviction_params); free(params->eviction_candidates); + free(params); cache_struct_free(cache); } @@ -273,6 +274,7 @@ static void RandomLRU_parse_params(cache_t *cache, params->n_samples = (int)strtol(value, &end, 0); } else if (strcasecmp(key, "print") == 0) { printf("current parameters: n-samples=%d\n", params->n_samples); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/S3FIFO.c b/libCacheSim/cache/eviction/S3FIFO.c index bcbde8a93..0a3288212 100644 --- a/libCacheSim/cache/eviction/S3FIFO.c +++ b/libCacheSim/cache/eviction/S3FIFO.c @@ -475,6 +475,7 @@ static void S3FIFO_parse_params(cache_t *cache, params->move_to_main_threshold = atoi(value); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", S3FIFO_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/S3FIFOd.c b/libCacheSim/cache/eviction/S3FIFOd.c index 79826721d..dad0e4330 100644 --- a/libCacheSim/cache/eviction/S3FIFOd.c +++ b/libCacheSim/cache/eviction/S3FIFOd.c @@ -185,6 +185,9 @@ static void S3FIFOd_free(cache_t *cache) { params->small_fifo->cache_free(params->small_fifo); params->ghost_fifo->cache_free(params->ghost_fifo); params->main_fifo->cache_free(params->main_fifo); + /* init also builds these two to track evicted objects */ + params->small_eviction->cache_free(params->small_eviction); + params->main_eviction->cache_free(params->main_eviction); free(cache->eviction_params); cache_struct_free(cache); } @@ -567,6 +570,7 @@ static void S3FIFOd_parse_params(cache_t *cache, params->move_to_main_threshold = atoi(value); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", S3FIFOd_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/S3FIFOv0.c b/libCacheSim/cache/eviction/S3FIFOv0.c index a9caf2a51..2bd01b988 100644 --- a/libCacheSim/cache/eviction/S3FIFOv0.c +++ b/libCacheSim/cache/eviction/S3FIFOv0.c @@ -478,8 +478,12 @@ static inline bool S3FIFOv0_can_insert(cache_t *cache, const request_t *req) { // *********************************************************************** static const char *S3FIFOv0_current_params(S3FIFOv0_params_t *params) { static __thread char params_str[128]; + /* main_fifo is only built after the parameters are parsed, so it is still + * NULL when the user asks for the parameters with `-e print`; it is always a + * plain FIFO in this variant */ snprintf(params_str, 128, "small-size-ratio=%.4lf,main-cache=%s\n", - params->small_size_ratio, params->main_fifo->cache_name); + params->small_size_ratio, + params->main_fifo == NULL ? "FIFO" : params->main_fifo->cache_name); return params_str; } @@ -510,6 +514,7 @@ static void S3FIFOv0_parse_params(cache_t *cache, params->move_to_main_threshold = atoi(value); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", S3FIFOv0_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/SLRU.c b/libCacheSim/cache/eviction/SLRU.c index ec84066e7..6390e00bd 100644 --- a/libCacheSim/cache/eviction/SLRU.c +++ b/libCacheSim/cache/eviction/SLRU.c @@ -487,6 +487,7 @@ static void SLRU_parse_params(cache_t *cache, } } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", SLRU_current_params(cache, params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/SLRUv0.c b/libCacheSim/cache/eviction/SLRUv0.c index 190c2ba79..78c4c358e 100644 --- a/libCacheSim/cache/eviction/SLRUv0.c +++ b/libCacheSim/cache/eviction/SLRUv0.c @@ -82,6 +82,7 @@ cache_t *SLRUv0_init(const common_cache_params_t ccache_params, cache->eviction_params = (SLRUv0_params_t *)malloc(sizeof(SLRUv0_params_t)); SLRUv0_params_t *params = (SLRUv0_params_t *)(cache->eviction_params); + memset(params, 0, sizeof(SLRUv0_params_t)); SLRUv0_parse_params(cache, DEFAULT_CACHE_PARAMS); if (cache_specific_params != NULL) { @@ -113,6 +114,7 @@ static void SLRUv0_free(cache_t *cache) { for (int i = 0; i < params->n_seg; i++) params->LRUs[i]->cache_free(params->LRUs[i]); free(params->LRUs); + free(params); cache_struct_free(cache); } @@ -373,6 +375,7 @@ static void SLRUv0_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", SLRUv0_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); @@ -396,7 +399,6 @@ static void SLRUv0_parse_params(cache_t *cache, */ static void SLRUv0_cool(cache_t *cache, const request_t *req, int i) { SLRUv0_params_t *params = (SLRUv0_params_t *)(cache->eviction_params); - request_t *saved_req = new_request(); cache_t *lru = params->LRUs[i]; // the last LRU is evict-only, do not move to a lower lru if (i == 0) { @@ -404,6 +406,9 @@ static void SLRUv0_cool(cache_t *cache, const request_t *req, int i) { return; }; + // only needed once we know the object is moving to a lower lru + request_t *saved_req = new_request(); + // the evicted object move to lower lru cache_obj_t *obj_evicted = lru->to_evict(lru, req); copy_cache_obj_to_request(saved_req, obj_evicted); diff --git a/libCacheSim/cache/eviction/Size.c b/libCacheSim/cache/eviction/Size.c index 4a7489502..9cc86ad1c 100644 --- a/libCacheSim/cache/eviction/Size.c +++ b/libCacheSim/cache/eviction/Size.c @@ -85,6 +85,7 @@ static void Size_free(cache_t *cache) { node = pqueue_pop(params->pq); } pqueue_free(params->pq); + my_free(sizeof(Size_params_t), params); cache_struct_free(cache); } diff --git a/libCacheSim/cache/eviction/TwoQ.c b/libCacheSim/cache/eviction/TwoQ.c index 31b8d6bfb..c58602be2 100644 --- a/libCacheSim/cache/eviction/TwoQ.c +++ b/libCacheSim/cache/eviction/TwoQ.c @@ -352,6 +352,7 @@ static void TwoQ_parse_params(cache_t *cache, params->Aout_size_ratio = strtod(value, NULL); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", TwoQ_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 891911259..0befbe65d 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -96,13 +96,11 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, cache->eviction_params = (WTinyLFU_params_t *)malloc(sizeof(WTinyLFU_params_t)); WTinyLFU_params_t *params = (WTinyLFU_params_t *)(cache->eviction_params); + memset(params, 0, sizeof(WTinyLFU_params_t)); - if (ccache_params.consider_obj_metadata) { - cache->obj_md_size = params->main_cache->obj_md_size; - // TODO: not sure whether it works - } else { - cache->obj_md_size = 0; - } + /* obj_md_size is set once main_cache exists; it is read from main_cache, + * which is only built further down */ + cache->obj_md_size = 0; WTinyLFU_parse_params(cache, DEFAULT_PARAMS); if (cache_specific_params != NULL) { @@ -144,6 +142,10 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, ERROR("WTinyLFU does not support %s \n", params->main_cache_type); } + if (ccache_params.consider_obj_metadata) { + cache->obj_md_size = params->main_cache->obj_md_size; + } + snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "WTinyLFU-w%.2lf-%s", params->window_size, params->main_cache_type); @@ -192,6 +194,7 @@ static void WTinyLFU_free(cache_t *cache) { minimalIncrementCBF_free(params->CBF); free(params->CBF); free_request(params->req_local); + free(params); cache_struct_free(cache); } @@ -346,6 +349,17 @@ static bool WTinyLFU_remove(cache_t *cache, obj_id_t obj_id) { return false; } +/* main_cache is only built after the parameters are parsed, so report the + * configured type, which is what `-e print` runs against */ +static const char *WTinyLFU_current_params(WTinyLFU_params_t *params) { + static __thread char params_str[128]; + snprintf(params_str, 128, "main-cache=%s,window-size=%.4lf", + params->main_cache == NULL ? params->main_cache_type + : params->main_cache->cache_name, + params->window_size); + return params_str; +} + static void WTinyLFU_parse_params(cache_t *cache, const char *cache_specific_params) { WTinyLFU_params_t *params = (WTinyLFU_params_t *)cache->eviction_params; @@ -353,6 +367,7 @@ static void WTinyLFU_parse_params(cache_t *cache, // params->max_request_num = 32 * cache->cache_size; // 32 * cache_size char *params_str = strdup(cache_specific_params); + char *old_params_str = params_str; while (params_str != NULL && params_str[0] != '\0') { /* different parameters are separated by comma, * key and value are separated by = */ @@ -372,12 +387,17 @@ static void WTinyLFU_parse_params(cache_t *cache, ERROR("window_size must be in [0, 1)\n"); exit(1); } + } else if (strcasecmp(key, "print") == 0) { + printf("current parameters: %s\n", WTinyLFU_current_params(params)); + free(old_params_str); + exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); + free(old_params_str); exit(1); } } - return; + free(old_params_str); } /* WTinyLFU cannot an object larger than segment size */ diff --git a/libCacheSim/cache/eviction/cpp/LRU_K.cpp b/libCacheSim/cache/eviction/cpp/LRU_K.cpp index 9dc290936..526877a7f 100644 --- a/libCacheSim/cache/eviction/cpp/LRU_K.cpp +++ b/libCacheSim/cache/eviction/cpp/LRU_K.cpp @@ -117,6 +117,7 @@ static void LRU_K_parse_params(cache_t *cache, lruk->k = static_cast(k_val); } else if (strcasecmp(key, "print") == 0) { printf("LRU_K parameters: k=%d\n", lruk->k); + free(to_free); exit(0); } else { ERROR("LRU_K does not have parameter %s\n", key); diff --git a/libCacheSim/cache/eviction/fifo/LP_ARC.c b/libCacheSim/cache/eviction/fifo/LP_ARC.c index 8f8ec0da8..ee7660c9e 100644 --- a/libCacheSim/cache/eviction/fifo/LP_ARC.c +++ b/libCacheSim/cache/eviction/fifo/LP_ARC.c @@ -504,6 +504,7 @@ static void LP_ARC_parse_params(cache_t *cache, if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", LP_ARC_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c index 85b2a0ce6..4fd31d299 100644 --- a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c @@ -406,6 +406,7 @@ static void LP_SFIFO_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", LP_SFIFO_current_params(cache, params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/fifo/LP_TwoQ.c b/libCacheSim/cache/eviction/fifo/LP_TwoQ.c index 15b6f50b5..f10ba071e 100644 --- a/libCacheSim/cache/eviction/fifo/LP_TwoQ.c +++ b/libCacheSim/cache/eviction/fifo/LP_TwoQ.c @@ -365,6 +365,7 @@ static void LP_TwoQ_parse_params(cache_t *cache, params->Aout_size_ratio = strtod(value, NULL); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", LP_TwoQ_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/fifo/SFIFO.c b/libCacheSim/cache/eviction/fifo/SFIFO.c index ad3d123ef..98390bc22 100644 --- a/libCacheSim/cache/eviction/fifo/SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/SFIFO.c @@ -393,6 +393,7 @@ static void SFIFO_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", SFIFO_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/fifo/SFIFOv0.c b/libCacheSim/cache/eviction/fifo/SFIFOv0.c index 4ee5a8db2..9a1a4f867 100644 --- a/libCacheSim/cache/eviction/fifo/SFIFOv0.c +++ b/libCacheSim/cache/eviction/fifo/SFIFOv0.c @@ -402,6 +402,7 @@ static void SFIFOv0_parse_params(cache_t *cache, } else if (strcasecmp(key, "print") == 0) { printf("current parameters: %s\n", SFIFOv0_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/other/S3LRU.c b/libCacheSim/cache/eviction/other/S3LRU.c index 7fe72adca..ed7550ebd 100644 --- a/libCacheSim/cache/eviction/other/S3LRU.c +++ b/libCacheSim/cache/eviction/other/S3LRU.c @@ -495,6 +495,7 @@ static void S3LRU_parse_params(cache_t *cache, params->promote_on_hit = atoi(value); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", S3LRU_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/other/flashProb.c b/libCacheSim/cache/eviction/other/flashProb.c index 980bf65f2..ed76a2f02 100644 --- a/libCacheSim/cache/eviction/other/flashProb.c +++ b/libCacheSim/cache/eviction/other/flashProb.c @@ -359,10 +359,13 @@ static inline int64_t flashProb_get_n_obj(const cache_t *cache) { // *********************************************************************** static const char *flashProb_current_params(flashProb_params_t *params) { static __thread char params_str[128]; - snprintf(params_str, 128, - "ram-size-ratio=%.4lf,disk-admit-prob=%.4lf,ram-cache=%s\n", - params->ram_size_ratio, params->disk_admit_prob, - params->ram->cache_name); + /* ram is only built after the parameters are parsed, so report the + * configured type, which is what `-e print` runs against */ + snprintf( + params_str, 128, + "ram-size-ratio=%.4lf,disk-admit-prob=%.4lf,ram-cache=%s\n", + params->ram_size_ratio, params->disk_admit_prob, + params->ram == NULL ? params->ram_cache_type : params->ram->cache_name); return params_str; } @@ -395,6 +398,7 @@ static void flashProb_parse_params(cache_t *cache, strncpy(params->disk_cache_type, value, 15); } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", flashProb_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/eviction/plugin_cache.c b/libCacheSim/cache/eviction/plugin_cache.c index 2362ca6b8..8b36b43ec 100644 --- a/libCacheSim/cache/eviction/plugin_cache.c +++ b/libCacheSim/cache/eviction/plugin_cache.c @@ -400,8 +400,8 @@ static void pluginCache_parse_params(cache_t *cache, char *key = strsep((char **)¶ms_str, "="); char *value = strsep((char **)¶ms_str, ","); - // Check if value is NULL - if (value == NULL) { + // Check if value is NULL; "print" is a bare flag, not a key=value pair + if (value == NULL && (key == NULL || strcasecmp(key, "print") != 0)) { ERROR("Parameter '%s' is missing a value in cache '%s'\n", key, cache->cache_name); exit(1); @@ -426,6 +426,7 @@ static void pluginCache_parse_params(cache_t *cache, params->cache_name = strdup(value); } else if (strcasecmp(key, "print") == 0) { printf("current parameters: plugin_path=%s\n", params->plugin_path); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); diff --git a/libCacheSim/cache/prefetch/Mithril.c b/libCacheSim/cache/prefetch/Mithril.c index 1fc94eb14..4cfffabc5 100644 --- a/libCacheSim/cache/prefetch/Mithril.c +++ b/libCacheSim/cache/prefetch/Mithril.c @@ -76,6 +76,7 @@ static void set_Mithril_default_init_params( static void Mithril_parse_init_params(const char *cache_specific_params, Mithril_init_params_t *init_params) { char *params_str = strdup(cache_specific_params); + char *old_params_str = params_str; while (params_str != NULL && params_str[0] != '\0') { char *key = strsep((char **)¶ms_str, "="); @@ -122,13 +123,16 @@ static void Mithril_parse_init_params(const char *cache_specific_params, } else if (strcasecmp(key, "print") == 0 || strcasecmp(key, "default") == 0) { printf("default params: %s\n", Mithril_default_params()); + free(old_params_str); exit(0); } else { ERROR("Mithril does not have parameter %s\n", key); printf("default params: %s\n", Mithril_default_params()); + free(old_params_str); exit(1); } } + free(old_params_str); } static void set_Mithril_params(Mithril_params_t *Mithril_params, diff --git a/libCacheSim/cache/prefetch/PG.c b/libCacheSim/cache/prefetch/PG.c index f3760c42e..931496b3d 100644 --- a/libCacheSim/cache/prefetch/PG.c +++ b/libCacheSim/cache/prefetch/PG.c @@ -56,6 +56,7 @@ static void set_PG_default_init_params(PG_init_params_t *init_params) { static void PG_parse_init_params(const char *cache_specific_params, PG_init_params_t *init_params) { char *params_str = strdup(cache_specific_params); + char *old_params_str = params_str; while (params_str != NULL && params_str[0] != '\0') { char *key = strsep((char **)¶ms_str, "="); @@ -74,13 +75,16 @@ static void PG_parse_init_params(const char *cache_specific_params, } else if (strcasecmp(key, "print") == 0 || strcasecmp(key, "default") == 0) { printf("default params: %s\n", PG_default_params()); + free(old_params_str); exit(0); } else { ERROR("pg does not have parameter %s\n", key); printf("default params: %s\n", PG_default_params()); + free(old_params_str); exit(1); } } + free(old_params_str); } static void set_PG_params(PG_params_t *PG_params, PG_init_params_t *init_params, diff --git a/test/test_cli.sh b/test/test_cli.sh index c27aea2ce..02d6d45c2 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -151,9 +151,69 @@ expect_output "qdlp -e print" "fifo-size-ratio=" \ expect_output "s3fifod -e print" "fifo-size-ratio=" \ "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral s3fifod 1gb -e print -for algo in arc car clock clockpro lecar lru-prob beladysize fifo-merge s3fifo 2q; do - expect_ok "${algo} -e print" \ - "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 1gb -e print +# Sweep every algorithm the CLI registers rather than a hand-picked list: the +# crashes this covers were found in variants that a shorter list missed +# (s3fifov0, flashProb). Algorithms behind an optional build flag report +# "do not support algorithm" and are skipped. +ALL_ALGOS="2q 3LCache CAR GLCache RandomLRU arc arcv0 cacheus clock clock2qplus + clockpro fifo fifo-merge fifo-reinsertion fifomerge flashProb gdsf gl-cache + lecar lecarv0 lfu lfucpp lfuda lhd lirs lrb lru lru-k lru-prob nop + pluginCache qdlp random randomTwo s3-fifo s3-fifov0 s3fifo s3fifod s3fifov0 + sieve size slru slruv0 twoq wtinyLFU" + +n_skipped=0 +for algo in ${ALL_ALGOS}; do + out=$("${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 1gb -e print 2>&1) + rc=$? + if [[ ${rc} -ne 0 ]] && grep -qi "do not support algorithm" <<<"${out}"; then + n_skipped=$((n_skipped + 1)) + continue + fi + if [[ ${rc} -eq 0 ]]; then + _report 0 "" + else + _report 1 "${algo} -e print (exit ${rc})" + echo "${out}" | tail -3 | sed 's/^/ /' + fi +done +echo " (${n_skipped} algorithms not compiled in, skipped)" + +echo "running per-algorithm replay tests" + +# Actually replay a trace with each algorithm, not just parse its parameters. +# Under the LeakSanitizer build CI uses, this is what catches allocations that +# init makes and free forgets — the `-e print` cases above exit early, so the +# cache is never torn down and a missing free stays invisible. +n_skipped=0 +for algo in ${ALL_ALGOS}; do + # pluginCache loads an eviction policy from an external .so that is not + # built here; see doc/quickstart_plugin.md + if [[ "${algo}" == "pluginCache" ]]; then + n_skipped=$((n_skipped + 1)) + continue + fi + out=$("${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 10mb \ + --num-req=20000 2>&1) + rc=$? + if [[ ${rc} -ne 0 ]] && grep -qi "do not support algorithm" <<<"${out}"; then + n_skipped=$((n_skipped + 1)) + continue + fi + if [[ ${rc} -eq 0 ]]; then + _report 0 "" + else + _report 1 "${algo} replay (exit ${rc})" + echo "${out}" | tail -3 | sed 's/^/ /' + fi +done +echo " (${n_skipped} algorithms not compiled in, skipped)" + +# Object metadata accounting reads from the sub-cache, which some algorithms +# only build partway through init. +for algo in wtinyLFU qdlp s3fifo slru lru; do + expect_ok "${algo} replay with --consider-obj-metadata=true" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 10mb \ + --num-req=20000 --consider-obj-metadata=true done echo "running SLRU parameter validation tests" From ab5507b69fc21e66dce3518ce61547d7cc8cab61 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 01:37:18 +0000 Subject: [PATCH 11/33] fix: undefined behaviour in SHARDS at a sample rate of 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex caught this on the previous commit and it is a real defect I introduced the explicit cast around without fixing. UINT64_MAX has no exact double representation — it rounds up to 2^64 — so at a sample rate of 1, `(double)UINT64_MAX * sample_rate` is 2^64, which does not fit in a uint64_t. The conversion is undefined before the `sample_rate == 1` branch below it can overwrite the result. Compiling the old form with -fsanitize=float-cast-overflow reports mrcProfiler.cpp:104:61: runtime error: 1.84467e+19 is outside the range of representable values of type 'long unsigned int' and the conversion silently yields 0. It went unnoticed because GCC's -fsanitize=undefined does not include that check by default, so the ubuntu job never saw it. Full sampling is now taken before scaling, so the conversion only runs on a rate below 1. That bound is enough on its own: 2^64 * (1 - 2^-53) is 2^64 - 2048, exactly representable and in range, so no clamp is needed. testCLI now covers SHARDS at rates 1, 0.999, 0.5 and 0.0001, and the FIX_SIZE mode. Not fixed, and now documented instead: `--profiler=MINISIM` aborts with "cannot load internal cache FIFO: undefined symbol: FIFO_init". It resolves eviction algorithms with dlsym() against the mrcProfiler executable, but those constructors are in the static library and nothing in mrcProfiler references them, so the linker never pulls them in and -rdynamic cannot help. It fails the same way on develop. Fixing it means either whole-archive linking or routing MINISIM through create_cache(), which is a design call for the maintainers, so doc/quickstart_mrcProfiler.md now warns instead of promising a command that aborts. Its examples also use ./bin/mrcProfiler now, matching the other guides. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/quickstart_mrcProfiler.md | 39 +++++++++++++------------ libCacheSim/mrcProfiler/mrcProfiler.cpp | 15 ++++++---- test/test_cli.sh | 19 ++++++++++++ 3 files changed, 50 insertions(+), 23 deletions(-) diff --git a/doc/quickstart_mrcProfiler.md b/doc/quickstart_mrcProfiler.md index 86b437a26..a8640b77d 100644 --- a/doc/quickstart_mrcProfiler.md +++ b/doc/quickstart_mrcProfiler.md @@ -18,12 +18,12 @@ First, [build libCacheSim](/doc/install.md). After building libCacheSim, `mrcPro ## Basic Usage ``` -./mrcProfiler trace_path trace_type --algo=[LRU] --profiler=[SHARDS|MINISIM] +./bin/mrcProfiler trace_path trace_type --algo=[LRU] --profiler=[SHARDS|MINISIM] --profiler-params=[FIX_RATE,0.01,hash_salt|FIX_SIZE,8192,hash_salt|FIX_RATE,0.01,thread_num(for MINISIM)] --size=[0.01,1,100|1MiB,100MiB,100|0.001,0.002,0.004,0.008,0.016|1MiB,10MiB,10MiB,1GiB] ``` -Use ./mrcProfiler --help for more details. +Use ./bin/mrcProfiler --help for more details. Plot scripts are provided in `scripts/profile_mrc.py`. See [here](/scripts/README.md) for more details. @@ -35,13 +35,13 @@ SHARDS is configured in `fixed sampling rate` mode with a sampling rate of `0.01 The cache sizes for MRC generation are specified in `fixed-size mode`, spanning `10` evenly spaced points from `100MB` to `1GB`: ```bash -./mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=100MB,1GB,10 +./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=100MB,1GB,10 ``` SHARDS can also operate in `fixed sample size` mode, limiting memory usage by sampling a fixed number of unique objects. The example below samples `2048` objects: ```bash -./mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_SIZE,2048,42 --size=100MB,1GB,10 +./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_SIZE,2048,42 --size=100MB,1GB,10 ``` ### Profiling MRC with WSS-Based Sizes @@ -49,7 +49,7 @@ SHARDS can also operate in `fixed sample size` mode, limiting memory usage by sa Generate an MRC based on WSS percentages. The example below creates `10` evenly spaced points from `10%` to `50%` of the WSS: ```bash -./mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 +./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 ``` ### Profiling MRC with Specific Sizes @@ -60,13 +60,13 @@ mrcProfiler supports both `WSS-based` and `fixed-size` MRC generation for specif **WSS-based sizes:** ```bash -./mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.01,0.02,0.04,0.08,0.16 +./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.01,0.02,0.04,0.08,0.16 ``` **Fixed cache sizes:** ```bash -./mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=10MB,20MB,40MB,80MB,160MB +./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=10MB,20MB,40MB,80MB,160MB ``` @@ -76,15 +76,18 @@ mrcProfiler supports both `WSS-based` and `fixed-size` MRC generation for specif In the example below, `FIX_RATE,0.01,10` sets a `1%` sampling rate and `10` threads. Note: Sampling rates above 0.5 disable sampling (full trace replay). ```bash -./mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=FIFO --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=0.1,0.5,10 +./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=FIFO --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=0.1,0.5,10 ``` +> [!WARNING] +> MINISIM currently aborts with `cannot load internal cache FIFO: undefined symbol: FIFO_init` in a standard build. It looks its eviction algorithms up with `dlsym()` against the `mrcProfiler` executable, but those constructors live in the static library and nothing in `mrcProfiler` references them, so the linker never pulls them in. Use `--profiler=SHARDS` in the meantime; it covers LRU. + ### Ignoring Object Sizes To ignore object sizes (treat all objects as 1-byte): ```bash -./mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 --ignore-obj-size +./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 --ignore-obj-size ``` ### Supporting Different Trace Formats @@ -105,13 +108,13 @@ Commands: ```bash # cachesim -time ./cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral LRU 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 +time ./bin/cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral LRU 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 # mrcProfiler with SHARDS with 0.01 sample rate -time ./mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 +time ./bin/mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 # mrcProfiler with SHARDS with 8192 sample size -time ./mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=LRU --profiler=SHARDS --profiler-params=FIX_SIZE,32768,10 --size=10MB,100MB,10 +time ./bin/mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=LRU --profiler=SHARDS --profiler-params=FIX_SIZE,32768,10 --size=10MB,100MB,10 ``` resluts: @@ -128,22 +131,22 @@ Commands: ```bash # cachesim for FIFO -time ./cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral FIFO 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 +time ./bin/cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral FIFO 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 # cachesim for ARC -time ./cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral ARC 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 +time ./bin/cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral ARC 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 # cachesim for S3FIFO -time ./cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral S3FIFO 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 +time ./bin/cachesim /path_to/cluster52.oracleGeneral.sample10 oracleGeneral S3FIFO 10MB,20MB,30MB,40MB,50MB,60MB,70MB,80MB,90MB,100MB --verbose=0 # mrcProfiler for FIFO eviction algorithm with MINISIM with 0.01 sample rate -time ./mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=FIFO --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 +time ./bin/mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=FIFO --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 # mrcProfiler for ARC eviction algorithm with MINISIM with 0.01 sample rate -time ./mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=ARC --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 +time ./bin/mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=ARC --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 # mrcProfiler for S3FIFO eviction algorithm with MINISIM with 0.01 sample rate -time ./mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=S3FIFO --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 +time ./bin/mrcProfiler /path_to/cluster52.oracleGeneral.sample10 oracleGeneral --algo=S3FIFO --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=10MB,100MB,10 ``` resluts: diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index ff92a3b77..df485f882 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -95,13 +95,18 @@ void mrcProfiler::MRCProfilerSHARDS::fixed_sample_rate_run() { double sample_rate = params_.shards_params.sample_rate; std::vector local_hit_cnt_vec(mrc_size_vec.size(), 0); std::vector local_hit_size_vec(mrc_size_vec.size(), 0); - /* UINT64_MAX has no exact double representation, so make the widening - * explicit; clang errors on the implicit form under -Werror */ - uint64_t sample_max = - static_cast(static_cast(UINT64_MAX) * sample_rate); - if (sample_rate == 1) { + /* UINT64_MAX has no exact double representation: it rounds up to 2^64, which + * is out of range for uint64_t. Take full sampling before scaling, so the + * conversion below only ever runs on a rate < 1, where the product is at most + * 2^64 - 2048 and converts cleanly. Doing it the other way round is undefined + * behaviour even though the result is immediately overwritten. */ + uint64_t sample_max; + if (sample_rate >= 1) { INFO("sample_rate is 1, no need to sample\n"); sample_max = UINT64_MAX; + } else { + sample_max = + static_cast(static_cast(UINT64_MAX) * sample_rate); } double sampled_cnt = 0, sampled_size = 0; int64_t current_time = 0; diff --git a/test/test_cli.sh b/test/test_cli.sh index 02d6d45c2..dac55a179 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -273,6 +273,25 @@ if [[ -x "${BIN_DIR}/mrcProfiler" ]]; then "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ --algo=LRU --profiler=SHARDS --profiler-params=FIX_RATE,0.01,42 --size=0.1,0.5,10 \ --ignore-obj-size + + # A sample rate of 1 means no sampling. UINT64_MAX rounds up to 2^64 as a + # double, so scaling by the rate before special-casing this converts a value + # that does not fit back into uint64_t. + for rate in 1 0.999 0.5 0.0001; do + expect_ok "mrcProfiler SHARDS at sample rate ${rate}" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo=LRU --profiler=SHARDS --profiler-params="FIX_RATE,${rate},42" \ + --size=0.1,0.5,10 + done + + # Not covered: --profiler=MINISIM. It resolves cache constructors with + # dlsym() against the mrcProfiler executable, but those live in the static + # library and nothing references them, so the linker never pulls them in and + # it aborts with "undefined symbol: FIFO_init". Broken on develop too. + + expect_ok "mrcProfiler SHARDS FIX_SIZE" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo=LRU --profiler=SHARDS --profiler-params=FIX_SIZE,2048,42 --size=100MB,1GB,10 fi echo From ea8969daaee09808df7619f353b0bde569fbbd65 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 01:47:18 +0000 Subject: [PATCH 12/33] docs: note WTinyLFU's shared object-metadata size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex flagged that cache->obj_md_size stands in for two sub-caches with different per-object overheads. The observation is right: WTinyLFU_can_insert() charges this value against params->LRU->cache_size, so with main-cache=FIFO it charges 0 against a window LRU that actually reserves 16 bytes per object, and admits objects the window cannot hold. Not changed. The assignment restores what the original code intended — before this branch it read main_cache before that pointer was assigned and segfaulted, so no working behaviour depends on either reading. Splitting the accounting (window metadata for window admission, main for transfers) changes admission decisions and therefore miss ratios, which is a modelling choice in WTinyLFU rather than a typo, so it is the maintainers' call and not something to slip into a polish PR. Comment records the asymmetry so it is not silently wrong. Verified comment-only: miss ratios for main-cache FIFO/LRU/SLRU with --consider-obj-metadata=true are unchanged, ctest 10/10 plain and LSan. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/cache/eviction/WTinyLFU.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 0befbe65d..5798af9de 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -143,6 +143,13 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, } if (ccache_params.consider_obj_metadata) { + /* NOTE: one value stands in for two sub-caches. WTinyLFU_can_insert() + * charges this against params->LRU->cache_size, so when the main policy's + * per-object metadata differs from the window LRU's the window check is + * off: main-cache=FIFO reports 0 while the window LRU charges 16 bytes. + * Splitting it (window metadata for window admission, main for transfers) + * would change admission decisions and therefore miss ratios, so it is + * left as the original accounting rather than changed in passing. */ cache->obj_md_size = params->main_cache->obj_md_size; } From 502776f7b2bf5b2f495a04b4ee2a7873e611826c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 02:34:18 +0000 Subject: [PATCH 13/33] fix: make the MINISIM profiler work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mrcProfiler --profiler=MINISIM` aborted on every run, on develop as well as here: cannot load internal cache FIFO: undefined symbol: FIFO_init It looked its eviction algorithm up with dlsym() against its own executable. That cannot work: the constructors sit in an archive member nothing references, so the linker never pulls them in and the symbol is absent from the binary however it is linked — -rdynamic does not help. Every MINISIM example in doc/quickstart_mrcProfiler.md failed. Moved the name to constructor mapping out of cachesim's private header and into the library, as cache/cacheAlgoRegistry.c. Referencing the table from a translation unit the profiler already links is what pulls those archive members in, so the lookup is a plain call with no dynamic loading, and it works the same on macOS. cachesim and MINISIM now share one table instead of one table and a dlsym path; create_cache() keeps the cases that need more than a lookup (a smaller hash table for hyperbolic, a default window size for tinyLFU, the oracle-trace checks for belady). Two error paths in plugin.c were unreachable or unhelpful: - create_cache_internal() aborted when dlsym failed, so create_cache_using_plugin()'s fallback to a shared library could never run. It returns NULL now and the fallback works. - create_cache_external() called exit() on a missing library, which the header documents as returning NULL, and printed a bare dlerror. An unknown algorithm reported "./libnosuchalgo.so: cannot open shared object file"; it now says which algorithm could not be created. Verified across FIFO, ARC, S3FIFO, LRU, sieve, lfu, twoq and clock, and that an unknown name fails with a message naming it. cachesim is unaffected: 28 deterministic algorithms produce identical miss ratios before and after. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/quickstart_mrcProfiler.md | 3 +- libCacheSim/bin/cachesim/cache_init.h | 112 +++++------------ libCacheSim/cache/CMakeLists.txt | 1 + libCacheSim/cache/cacheAlgoRegistry.c | 119 ++++++++++++++++++ libCacheSim/cache/plugin.c | 48 ++++--- .../include/libCacheSim/evictionAlgo.h | 31 +++++ 6 files changed, 211 insertions(+), 103 deletions(-) create mode 100644 libCacheSim/cache/cacheAlgoRegistry.c diff --git a/doc/quickstart_mrcProfiler.md b/doc/quickstart_mrcProfiler.md index a8640b77d..1fc93b67e 100644 --- a/doc/quickstart_mrcProfiler.md +++ b/doc/quickstart_mrcProfiler.md @@ -79,8 +79,7 @@ In the example below, `FIX_RATE,0.01,10` sets a `1%` sampling rate and `10` thre ./bin/mrcProfiler ../data/cloudPhysicsIO.vscsi vscsi --algo=FIFO --profiler=MINISIM --profiler-params=FIX_RATE,0.01,10 --size=0.1,0.5,10 ``` -> [!WARNING] -> MINISIM currently aborts with `cannot load internal cache FIFO: undefined symbol: FIFO_init` in a standard build. It looks its eviction algorithms up with `dlsym()` against the `mrcProfiler` executable, but those constructors live in the static library and nothing in `mrcProfiler` references them, so the linker never pulls them in. Use `--profiler=SHARDS` in the meantime; it covers LRU. +`--algo` accepts the same names as `cachesim`, so any built-in algorithm works — ARC, S3FIFO, sieve, twoq, and the rest. See the [README](/README.md#supported-algorithms) for the full list. ### Ignoring Object Sizes diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index bb7c243cd..44af3694b 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -14,99 +14,40 @@ extern "C" { #endif +/* log2 of the hash table size; 24 gives 16M entries */ +#define DEFAULT_HASHPOWER 24 + +/** + * @brief create a cache for the CLI, given the algorithm name + * + * @param hashpower log2 of the hash table size. This used to be adjusted by + * sniffing the trace path for "data/trace.", a file that has not existed for a + * long time, so the adjustment never fired. It is a --hashpower option now + * rather than a hidden rule, because sampling-based algorithms draw candidates + * from the hash table and so their miss ratios depend on its size — not + * something to change silently based on where a trace happens to live. + */ static inline cache_t *create_cache(const char *trace_path, const char *eviction_algo, const uint64_t cache_size, const char *eviction_params, - const bool consider_obj_metadata) { + const bool consider_obj_metadata, + const int hashpower) { common_cache_params_t cc_params = { .cache_size = cache_size, .default_ttl = 86400 * 300, - .hashpower = 24, + .hashpower = hashpower, .consider_obj_metadata = consider_obj_metadata, }; cache_t *cache; - /* NOTE: there used to be a heuristic here shrinking hashpower by 8 when the - * trace path contained "data/trace.", to save memory on the sample traces. - * No such file has existed for a long time, so it never fired. Pointing it at - * the current sample traces is not a free fix: a smaller hash table changes - * which candidates sampling-based algorithms (RandomLRU, Hyperbolic, ...) - * draw, so miss ratios shift. Left out rather than silently changing results; - * re-add deliberately if the memory saving is worth that. */ - - typedef struct { - const char *name; - cache_t *(*init_func)(common_cache_params_t, const char *); - } eviction_algo_entry_t; - static const eviction_algo_entry_t simple_algos[] = { - {"2q", TwoQ_init}, - {"arc", ARC_init}, - {"arcv0", ARCv0_init}, - {"CAR", CAR_init}, - {"cacheus", Cacheus_init}, - {"clock", Clock_init}, - {"clock2qplus", Clock2QPlus_init}, - {"clockpro", ClockPro_init}, - {"fifo", FIFO_init}, - {"fifo-merge", FIFO_Merge_init}, - {"fifo-reinsertion", Clock_init}, - {"fifomerge", FIFO_Merge_init}, - {"flashProb", flashProb_init}, - {"gdsf", GDSF_init}, - {"lhd", LHD_init}, - {"lecar", LeCaR_init}, - {"lecarv0", LeCaRv0_init}, - {"lfu", LFU_init}, - {"lfucpp", LFUCpp_init}, - {"lfuda", LFUDA_init}, - {"lirs", LIRS_init}, - {"lru", LRU_init}, - {"lru-k", LRU_K_init}, - {"lru-prob", LRU_Prob_init}, - {"nop", nop_init}, - // plugin cache that allows user to implement custom cache - {"pluginCache", pluginCache_init}, - {"qdlp", QDLP_init}, - {"random", Random_init}, - {"RandomLRU", RandomLRU_init}, - {"randomTwo", RandomTwo_init}, - {"s3-fifo", S3FIFO_init}, - {"s3-fifov0", S3FIFOv0_init}, - {"s3fifo", S3FIFO_init}, - {"s3fifod", S3FIFOd_init}, - {"s3fifov0", S3FIFOv0_init}, - {"sieve", Sieve_init}, - {"size", Size_init}, - {"slru", SLRU_init}, - {"slruv0", SLRUv0_init}, - {"twoq", TwoQ_init}, - {"wtinyLFU", WTinyLFU_init}, -#ifdef ENABLE_3L_CACHE - {"3LCache", ThreeLCache_init}, -#endif -#ifdef ENABLE_GLCACHE - {"GLCache", GLCache_init}, - {"gl-cache", GLCache_init}, -#endif -#ifdef ENABLE_LRB - {"lrb", LRB_init}, -#endif - }; - - cache_t *(*init_func)(common_cache_params_t, const char *) = NULL; - for (size_t i = 0; i < sizeof(simple_algos) / sizeof(simple_algos[0]); ++i) { - if (strcasecmp(eviction_algo, simple_algos[i].name) == 0) { - init_func = simple_algos[i].init_func; - break; - } - } - - // Initializing for algorithms which require special handling (not in - // simple_algos) - if (init_func) { - cache = init_func(cc_params, eviction_params); - } else if (strcasecmp(eviction_algo, "hyperbolic") == 0) { + /* The name to constructor mapping lives in the library + * (cache/cacheAlgoRegistry.c) so that the MINISIM profiler, which only knows + * the algorithm by name, shares one table with the CLI. The cases below need + * more than a lookup — a smaller hash table, a default parameter, or a check + * that the trace carries the future information the algorithm needs — so + * they are handled here rather than in the registry. */ + if (strcasecmp(eviction_algo, "hyperbolic") == 0) { cc_params.hashpower = MAX(cc_params.hashpower - 8, 16); cache = Hyperbolic_init(cc_params, eviction_params); } else if (strcasecmp(eviction_algo, "tinyLFU") == 0) { @@ -154,8 +95,11 @@ static inline cache_t *create_cache(const char *trace_path, cc_params.hashpower = MAX(cc_params.hashpower - 8, 16); cache = BeladySize_init(cc_params, eviction_params); } else { - ERROR("do not support algorithm %s\n", eviction_algo); - abort(); + cache = create_cache_by_name(eviction_algo, cc_params, eviction_params); + if (cache == NULL) { + ERROR("do not support algorithm %s\n", eviction_algo); + abort(); + } } return cache; diff --git a/libCacheSim/cache/CMakeLists.txt b/libCacheSim/cache/CMakeLists.txt index 0d1ef8546..5bfe6c9c6 100644 --- a/libCacheSim/cache/CMakeLists.txt +++ b/libCacheSim/cache/CMakeLists.txt @@ -134,6 +134,7 @@ set(cache_sources_c ${eviction_sources_c} ${prefetch_sources_c} cache.c + cacheAlgoRegistry.c plugin.c ) diff --git a/libCacheSim/cache/cacheAlgoRegistry.c b/libCacheSim/cache/cacheAlgoRegistry.c new file mode 100644 index 000000000..3c6c8b965 --- /dev/null +++ b/libCacheSim/cache/cacheAlgoRegistry.c @@ -0,0 +1,119 @@ +/** + * @file cacheAlgoRegistry.c + * @brief Maps eviction algorithm names to their constructors. + * + * Callers that only have the algorithm's name — the CLI tools and the MINISIM + * profiler — used to find the constructor two different ways: cachesim carried + * its own table, while the profiler went through dlsym() against the running + * executable. The latter cannot work for a statically linked build, because the + * constructors live in an archive member nothing references, so the linker + * never pulls them in and the lookup fails at run time. + * + * Referencing the table from this translation unit is what pulls those archive + * members in, so the lookup is a plain function call with no dynamic loading. + */ + +#include + +#include "libCacheSim/cache.h" +#include "libCacheSim/evictionAlgo.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + const char *name; + cache_t *(*init_func)(common_cache_params_t, const char *); +} cache_algo_entry_t; + +/* Keep alphabetical; several names are aliases for the same constructor. */ +static const cache_algo_entry_t g_cache_algos[] = { + {"2q", TwoQ_init}, + {"arc", ARC_init}, + {"arcv0", ARCv0_init}, + {"CAR", CAR_init}, + {"cacheus", Cacheus_init}, + {"clock", Clock_init}, + {"clock2qplus", Clock2QPlus_init}, + {"clockpro", ClockPro_init}, + {"fifo", FIFO_init}, + {"fifo-merge", FIFO_Merge_init}, + {"fifo-reinsertion", Clock_init}, + {"fifomerge", FIFO_Merge_init}, + {"flashProb", flashProb_init}, + {"gdsf", GDSF_init}, + {"lhd", LHD_init}, + {"lecar", LeCaR_init}, + {"lecarv0", LeCaRv0_init}, + {"lfu", LFU_init}, + {"lfucpp", LFUCpp_init}, + {"lfuda", LFUDA_init}, + {"lirs", LIRS_init}, + {"lru", LRU_init}, + {"lru-k", LRU_K_init}, + {"lru-prob", LRU_Prob_init}, + {"nop", nop_init}, + /* plugin cache that allows user to implement custom cache */ + {"pluginCache", pluginCache_init}, + {"qdlp", QDLP_init}, + {"random", Random_init}, + {"RandomLRU", RandomLRU_init}, + {"randomTwo", RandomTwo_init}, + {"s3-fifo", S3FIFO_init}, + {"s3-fifov0", S3FIFOv0_init}, + {"s3fifo", S3FIFO_init}, + {"s3fifod", S3FIFOd_init}, + {"s3fifov0", S3FIFOv0_init}, + {"sieve", Sieve_init}, + {"size", Size_init}, + {"slru", SLRU_init}, + {"slruv0", SLRUv0_init}, + {"twoq", TwoQ_init}, + {"wtinyLFU", WTinyLFU_init}, + /* these need future information and are only valid on oracle traces, so + * callers that know the trace type should check before using them */ + {"belady", Belady_init}, + {"beladySize", BeladySize_init}, + {"hyperbolic", Hyperbolic_init}, +#ifdef ENABLE_3L_CACHE + {"3LCache", ThreeLCache_init}, +#endif +#ifdef ENABLE_GLCACHE + {"GLCache", GLCache_init}, + {"gl-cache", GLCache_init}, +#endif +#ifdef ENABLE_LRB + {"lrb", LRB_init}, +#endif +}; + +cache_init_func_ptr find_cache_init_func(const char *cache_algo_name) { + if (cache_algo_name == NULL) { + return NULL; + } + + for (size_t i = 0; i < sizeof(g_cache_algos) / sizeof(g_cache_algos[0]); + i++) { + if (strcasecmp(cache_algo_name, g_cache_algos[i].name) == 0) { + return g_cache_algos[i].init_func; + } + } + + return NULL; +} + +cache_t *create_cache_by_name(const char *cache_algo_name, + const common_cache_params_t cc_params, + const char *cache_specific_params) { + cache_init_func_ptr init_func = find_cache_init_func(cache_algo_name); + if (init_func == NULL) { + return NULL; + } + + return init_func(cc_params, cache_specific_params); +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/cache/plugin.c b/libCacheSim/cache/plugin.c index 4751aae32..4f0e9fb68 100644 --- a/libCacheSim/cache/plugin.c +++ b/libCacheSim/cache/plugin.c @@ -23,13 +23,18 @@ cache_t *create_cache_external(const char *const cache_alg_name, char shared_lib_path[256]; char cache_init_func_name[256]; - sprintf(shared_lib_path, "./lib%s.so", cache_alg_name); - sprintf(cache_init_func_name, "%s_init", cache_alg_name); - + snprintf(shared_lib_path, sizeof(shared_lib_path), "./lib%s.so", + cache_alg_name); + snprintf(cache_init_func_name, sizeof(cache_init_func_name), "%s_init", + cache_alg_name); + + /* Failure returns NULL, as the header documents, so the caller can report + * which algorithm it could not find. Exiting here instead made that + * reporting unreachable and left the user with a bare dlerror string. */ handle = dlopen(shared_lib_path, RTLD_LAZY); if (!handle) { - fprintf(stderr, "%s\n", dlerror()); - exit(EXIT_FAILURE); + WARN("cannot load %s: %s\n", shared_lib_path, dlerror()); + return NULL; } dlerror(); /* Clear any existing error */ @@ -43,8 +48,9 @@ cache_t *create_cache_external(const char *const cache_alg_name, cache_init = dlsym_ptr.func_ptr; if ((error = dlerror()) != NULL) { - fprintf(stderr, "%s\n", error); - exit(EXIT_FAILURE); + WARN("cannot find %s in %s: %s\n", cache_init_func_name, shared_lib_path, + error); + return NULL; } else { INFO("external cache %s loaded\n", cache_alg_name); } @@ -59,15 +65,25 @@ cache_t *create_cache_external(const char *const cache_alg_name, cache_t *create_cache_internal(const char *const cache_alg_name, common_cache_params_t cc_params, void *cache_specific_params) { - cache_t *(*cache_init)(common_cache_params_t, void *) = NULL; - char *err = NULL; + /* Built-in algorithms are looked up in the registry rather than through + * dlsym(). Their constructors live in an archive member that nothing else + * references, so in a statically linked build the linker never pulls them in + * and dlsym() cannot find them however the executable is linked. */ + cache_t *cache = create_cache_by_name(cache_alg_name, cc_params, + (const char *)cache_specific_params); + if (cache != NULL) { + return cache; + } + /* Fall back to dlsym for an algorithm that is not built in, e.g. one loaded + * into the process from elsewhere. */ char cache_init_func_name[256]; void *handle = dlopen(NULL, RTLD_GLOBAL); /* should not check err here, otherwise ubuntu will report err even though * everything is OK */ - sprintf(cache_init_func_name, "%s_init", cache_alg_name); + snprintf(cache_init_func_name, sizeof(cache_init_func_name), "%s_init", + cache_alg_name); // ISO C compliant way to convert void* to function pointer union { @@ -76,18 +92,16 @@ cache_t *create_cache_internal(const char *const cache_alg_name, } dlsym_ptr; dlsym_ptr.obj_ptr = dlsym(handle, cache_init_func_name); - cache_init = dlsym_ptr.func_ptr; - - err = dlerror(); + cache_t *(*cache_init)(common_cache_params_t, void *) = dlsym_ptr.func_ptr; if (cache_init == NULL) { - WARN("cannot load internal cache %s: error %s\n", cache_alg_name, err); - abort(); + /* Not an error yet: the caller falls back to loading a shared library. */ + (void)dlerror(); + return NULL; } INFO("internal cache %s loaded\n", cache_alg_name); - cache_t *cache = cache_init(cc_params, cache_specific_params); - return cache; + return cache_init(cc_params, cache_specific_params); } cache_t *create_cache_using_plugin(const char *const cache_alg_name, diff --git a/libCacheSim/include/libCacheSim/evictionAlgo.h b/libCacheSim/include/libCacheSim/evictionAlgo.h index 922c65e46..ad98787a9 100644 --- a/libCacheSim/include/libCacheSim/evictionAlgo.h +++ b/libCacheSim/include/libCacheSim/evictionAlgo.h @@ -197,6 +197,37 @@ cache_t *GLCache_init(const common_cache_params_t ccache_params, #endif +// *********************************************************************** +// **** **** +// **** lookup by algorithm name **** +// **** **** +// *********************************************************************** + +/** + * @brief look up the constructor for a built-in eviction algorithm + * + * The name is matched case-insensitively and accepts the same aliases as the + * command-line tools, e.g. "s3fifo", "s3-fifo". + * + * @param cache_algo_name algorithm name, may be NULL + * @return the constructor, or NULL if the name is not a built-in algorithm or + * was not compiled in (GLCache, LRB and 3LCache are behind build flags) + */ +cache_init_func_ptr find_cache_init_func(const char *cache_algo_name); + +/** + * @brief construct a built-in eviction algorithm by name + * + * Note that belady and beladySize need future information, so they only work on + * oracle traces; callers that know the trace type should check first. + * + * @return the new cache, or NULL if the name is not a built-in algorithm. The + * caller owns the result and frees it with cache->cache_free(). + */ +cache_t *create_cache_by_name(const char *cache_algo_name, + const common_cache_params_t cc_params, + const char *cache_specific_params); + #ifdef __cplusplus } #endif From 355a1f9f5a097c13f28a2244404b69c91040b4d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 02:34:43 +0000 Subject: [PATCH 14/33] feat: add --hashpower instead of guessing from the trace path cachesim shrank its hash table by 8 powers of two when the trace path contained "data/trace.". No such file has existed for a long time, so the saving never happened, and simply repointing the string at the current sample traces is not equivalent: sampling-based algorithms draw eviction candidates from the hash table, so RandomLRU and hyperbolic move when it resizes. Sizing a table by sniffing a filename is the wrong mechanism for something that changes results. Replaced with --hashpower, log2 of the table size, default 24 as before. The saving it was after is real and now available on any trace: replaying the sample trace goes from 106 MB to 15 MB at 20 and 8 MB at 16. Values outside 1..39 are rejected rather than silently ignored by cache_struct_init. Default behaviour is unchanged, since the heuristic never fired. The caveat about sampling-based algorithms is in --help and in doc/quickstart_cachesim.md so the trade-off is visible at the point of use. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/quickstart_cachesim.md | 8 ++++++++ libCacheSim/bin/MRC/parser_mini.c | 7 ++++--- libCacheSim/bin/cachesim/cli_parser.c | 16 +++++++++++++++- libCacheSim/bin/cachesim/internal.h | 1 + 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/doc/quickstart_cachesim.md b/doc/quickstart_cachesim.md index c4c5bd58e..0fb6735b6 100644 --- a/doc/quickstart_cachesim.md +++ b/doc/quickstart_cachesim.md @@ -193,4 +193,12 @@ You can use `-p` or `--prefetch` to set the prefetching algorithm. # Disable the print of the first few requests ./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 1gb --print-head-req=false + +# size the hash table: --hashpower is log2 of the number of entries, default 24 +# (16M). Lowering it cuts memory substantially on small traces — replaying the +# sample trace drops from about 106 MB to 8 MB at --hashpower=16 +./bin/cachesim ../data/cloudPhysicsIO.vscsi vscsi lru 1gb --hashpower=16 ``` + +> [!NOTE] +> Sampling-based algorithms such as `RandomLRU` and `hyperbolic` draw eviction candidates from the hash table, so their miss ratios shift slightly with `--hashpower`. Keep it fixed when comparing results. diff --git a/libCacheSim/bin/MRC/parser_mini.c b/libCacheSim/bin/MRC/parser_mini.c index 741319011..4e72855e2 100644 --- a/libCacheSim/bin/MRC/parser_mini.c +++ b/libCacheSim/bin/MRC/parser_mini.c @@ -332,9 +332,10 @@ void parse_mini_cmd(int argc, char *argv[], struct MINI_arguments *args) { for (int i = 0; i < args->n_eviction_algo; i++) { for (int j = 0; j < args->n_cache_size; j++) { int idx = i * args->n_cache_size + j; - args->caches[idx] = create_cache( - args->trace_path, args->eviction_algo[i], args->cache_sizes[j], - args->eviction_params, args->consider_obj_metadata); + args->caches[idx] = + create_cache(args->trace_path, args->eviction_algo[i], + args->cache_sizes[j], args->eviction_params, + args->consider_obj_metadata, DEFAULT_HASHPOWER); if (args->admission_algo != NULL) { args->caches[idx]->admissioner = diff --git a/libCacheSim/bin/cachesim/cli_parser.c b/libCacheSim/bin/cachesim/cli_parser.c index a7586322d..448c45112 100644 --- a/libCacheSim/bin/cachesim/cli_parser.c +++ b/libCacheSim/bin/cachesim/cli_parser.c @@ -46,6 +46,7 @@ enum argp_option_short { OPTION_PREFETCH_ALGO = 'p', OPTION_PREFETCH_PARAMS = 0x109, OPTION_PRINT_HEAD_REQ = 0x10a, + OPTION_HASHPOWER = 0x10b, }; /* @@ -94,6 +95,11 @@ static struct argp_option options[] = { {"verbose", OPTION_VERBOSE, "1", 0, "Produce verbose output", 10}, {"print-head-req", OPTION_PRINT_HEAD_REQ, "false", 0, "Print the first few requests", 10}, + {"hashpower", OPTION_HASHPOWER, "24", 0, + "Log2 of the hash table size, default 24 (16M entries). Lower it to save " + "memory on small traces. Note that sampling-based algorithms draw " + "candidates from the hash table, so their miss ratios depend on this", + 10}, {0, 0, 0, 0, 0, 0}}; @@ -163,6 +169,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case OPTION_CONSIDER_OBJ_METADATA: arguments->consider_obj_metadata = is_true(arg) ? true : false; break; + case OPTION_HASHPOWER: + arguments->hashpower = atoi(arg); + if (arguments->hashpower <= 0 || arguments->hashpower >= 40) { + ERROR("hashpower must be between 1 and 39, got %d\n", + arguments->hashpower); + } + break; case OPTION_WARMUP_SEC: arguments->warmup_sec = atoi(arg); break; @@ -226,6 +239,7 @@ static void init_arg(struct arguments *args) { args->use_ttl = false; args->ignore_obj_size = false; args->consider_obj_metadata = false; + args->hashpower = DEFAULT_HASHPOWER; args->report_interval = 3600 * 24; args->n_thread = n_cores(); args->warmup_sec = -1; @@ -347,7 +361,7 @@ void parse_cmd(int argc, char *argv[], struct arguments *args) { int idx = i * args->n_cache_size + j; args->caches[idx] = create_cache( args->trace_path, args->eviction_algo[i], args->cache_sizes[j], - args->eviction_params, args->consider_obj_metadata); + args->eviction_params, args->consider_obj_metadata, args->hashpower); if (args->admission_algo != NULL) { args->caches[idx]->admissioner = diff --git a/libCacheSim/bin/cachesim/internal.h b/libCacheSim/bin/cachesim/internal.h index 9b43e7717..f6db49031 100644 --- a/libCacheSim/bin/cachesim/internal.h +++ b/libCacheSim/bin/cachesim/internal.h @@ -47,6 +47,7 @@ struct arguments { bool consider_obj_metadata; bool use_ttl; bool print_head_req; + int hashpower; /* arguments generated */ reader_t *reader; From 61d94d76cd8a10c3075b4f65e3d5f4209bae258b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 02:34:43 +0000 Subject: [PATCH 15/33] fix: charge WTinyLFU's window its own object metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raised in review by Codex. WTinyLFU_can_insert() decided whether an object fits in the window by charging cache->obj_md_size, which holds the *main* cache's per-object overhead, against the window LRU's size. The two disagree: LRU and SLRU reserve 16 bytes, FIFO and Clock none. With main-cache=FIFO the window was charged nothing for an overhead it does reserve, so objects were admitted that the window could not hold. Each sub-cache is now checked against its own overhead — the window against the window LRU's, the main cache through its own can_insert(). The parent's obj_md_size, which cache_can_insert_default() uses for the whole-cache size check, takes the larger of the two: an object that does not fit under the heavier policy does not fit in this cache. Only reachable since the fix earlier on this branch, because --consider-obj-metadata=true segfaulted here on develop. With metadata off every object metadata size is zero and nothing changes; verified identical for main-cache FIFO/LRU/SLRU/sieve/ARC/clock. With metadata on, only the configurations whose overheads differ from the window's move: FIFO 0.8619 -> 0.8070 and clock 0.7732 -> 0.7733, while LRU, SLRU, sieve and ARC are unchanged. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/cache/eviction/WTinyLFU.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 5798af9de..38d8e7205 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -143,14 +143,14 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, } if (ccache_params.consider_obj_metadata) { - /* NOTE: one value stands in for two sub-caches. WTinyLFU_can_insert() - * charges this against params->LRU->cache_size, so when the main policy's - * per-object metadata differs from the window LRU's the window check is - * off: main-cache=FIFO reports 0 while the window LRU charges 16 bytes. - * Splitting it (window metadata for window admission, main for transfers) - * would change admission decisions and therefore miss ratios, so it is - * left as the original accounting rather than changed in passing. */ - cache->obj_md_size = params->main_cache->obj_md_size; + /* The window and the main cache can charge different per-object overheads + * (LRU and SLRU reserve 16 bytes, FIFO none), so neither value alone + * describes the pair. This one is what the parent-level size check in + * cache_can_insert_default() uses, so take the larger of the two: an object + * that does not fit under the heavier policy does not fit in this cache. + * WTinyLFU_can_insert() checks each sub-cache against its own overhead. */ + cache->obj_md_size = + MAX(params->LRU->obj_md_size, params->main_cache->obj_md_size); } snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "WTinyLFU-w%.2lf-%s", @@ -412,8 +412,12 @@ bool WTinyLFU_can_insert(cache_t *cache, const request_t *req) { WTinyLFU_params_t *params = (WTinyLFU_params_t *)cache->eviction_params; bool can_insert = cache_can_insert_default(cache, req); + /* An object enters through the window, so the window's own per-object + * overhead decides whether it fits there — not the main cache's, which can + * differ. The main cache checks itself with its own overhead. */ return can_insert && - (req->obj_size + cache->obj_md_size <= params->LRU->cache_size) && + (req->obj_size + params->LRU->obj_md_size <= + params->LRU->cache_size) && (params->main_cache->can_insert(params->main_cache, req)); } From 37219679f866f7099e4d0c323d857dc3e731e899 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 02:34:43 +0000 Subject: [PATCH 16/33] test: cover MINISIM, --hashpower and WTinyLFU metadata testCLI grows to 142 checks: - MINISIM across FIFO, ARC, S3FIFO, sieve, twoq, clock and lfu, which every one of aborted before, plus an unknown algorithm failing cleanly rather than crashing - --hashpower at 24/20/16/12, and rejection of 0, -1, 40 and 99 - wtinyLFU with metadata on for each main-cache type, since the window and main cache reserve different per-object overheads Passes under both a plain Release build and the LeakSanitizer build CI uses. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- test/test_cli.sh | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/test/test_cli.sh b/test/test_cli.sh index dac55a179..c32ecdc12 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -216,6 +216,30 @@ for algo in wtinyLFU qdlp s3fifo slru lru; do --num-req=20000 --consider-obj-metadata=true done +# WTinyLFU holds a window and a main cache whose per-object overheads differ +# (LRU and SLRU reserve 16 bytes, FIFO and Clock none), so each admission check +# has to use its own. +for main in FIFO LRU SLRU sieve ARC clock; do + expect_ok "wtinyLFU main-cache=${main} with metadata" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral wtinyLFU 10mb \ + --num-req=20000 -e "main-cache=${main}" --consider-obj-metadata=true +done + +echo "running hash table sizing tests" + +# --hashpower replaces a heuristic that keyed off the trace path. Sizing the +# table down is the point of it, so check the range is usable and validated. +for hp in 24 20 16 12; do + expect_output "cachesim --hashpower=${hp}" "miss ratio" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral lru 10mb \ + --num-req=20000 "--hashpower=${hp}" +done +for hp in 0 -1 40 99; do + expect_clean_error "cachesim rejects --hashpower=${hp}" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral lru 10mb \ + --num-req=20000 "--hashpower=${hp}" +done + echo "running SLRU parameter validation tests" # n-seg divides the cache size and the reported percentages, and seg-size fills @@ -284,10 +308,21 @@ if [[ -x "${BIN_DIR}/mrcProfiler" ]]; then --size=0.1,0.5,10 done - # Not covered: --profiler=MINISIM. It resolves cache constructors with - # dlsym() against the mrcProfiler executable, but those live in the static - # library and nothing references them, so the linker never pulls them in and - # it aborts with "undefined symbol: FIFO_init". Broken on develop too. + # MINISIM looks its eviction algorithm up by name. It used to do that with + # dlsym() against this executable, which cannot work when the constructors + # sit in an unreferenced archive member, so every run aborted with + # "undefined symbol: FIFO_init". Cover the non-LRU algorithms it exists for. + for algo in FIFO ARC S3FIFO sieve twoq clock lfu; do + expect_ok "mrcProfiler MINISIM with ${algo}" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo="${algo}" --profiler=MINISIM --profiler-params=FIX_RATE,0.01,4 \ + --size=0.1,0.5,10 + done + + expect_clean_error "mrcProfiler MINISIM with an unknown algorithm" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo=nosuchalgo --profiler=MINISIM --profiler-params=FIX_RATE,0.01,4 \ + --size=0.1,0.5,10 expect_ok "mrcProfiler SHARDS FIX_SIZE" \ "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ From 9007e886760e7e67a3783725cf7c4985eccf3e0c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 02:47:22 +0000 Subject: [PATCH 17/33] fix: floor the hash power composite caches give their sub-caches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raised in review by Codex against --hashpower. Several composite policies size their sub-caches by subtracting from the parent's hash power without a lower bound, and cache_struct_init treats a value of zero or less as "unset" and substitutes the full-size default. Asking for a small table therefore allocated several large ones: slruv0 --hashpower=4 -> 18 MB slruv0 --hashpower=5 -> 6 MB Cacheus, S3FIFOd, SLRUv0 and LP_SFIFO now clamp at 4, matching what Hyperbolic, Random, RandomTwo and RandomLRU already do. Behaviour is unchanged wherever the old arithmetic stayed positive — the clamp only binds below hash power 8 — and verified identical at the default for cacheus, s3fifod, slruv0, lru and fifo. Memory now falls monotonically as the hash power drops, 6 MB at 4 through 8 MB at 24. SFIFOv0 divides by that same expression rather than assigning it, so it divided by zero at hash power 4. Guarded the divisor instead of rewriting it to match its siblings: the division looks deliberate, and the algorithm is not reachable from the CLI, so there is no call to change its sizing on a guess. Also added the NaN sample rate case from #325 to testCLI, which arrived with the fix but without a test. testCLI is now 162 checks. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/cache/eviction/Cacheus.c | 2 +- libCacheSim/cache/eviction/S3FIFOd.c | 2 +- libCacheSim/cache/eviction/SLRUv0.c | 3 ++- libCacheSim/cache/eviction/fifo/LP_SFIFO.c | 2 +- libCacheSim/cache/eviction/fifo/SFIFOv0.c | 6 +++++- test/test_cli.sh | 22 ++++++++++++++++++++++ 6 files changed, 32 insertions(+), 5 deletions(-) diff --git a/libCacheSim/cache/eviction/Cacheus.c b/libCacheSim/cache/eviction/Cacheus.c index 30b86be3f..50d0ba47a 100644 --- a/libCacheSim/cache/eviction/Cacheus.c +++ b/libCacheSim/cache/eviction/Cacheus.c @@ -72,7 +72,7 @@ cache_t *Cacheus_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { common_cache_params_t updated_cc_params = ccache_params; /* reduce the hash table size */ - updated_cc_params.hashpower -= 2; + updated_cc_params.hashpower = MAX(4, updated_cc_params.hashpower - 2); cache_t *cache = cache_struct_init("Cacheus", updated_cc_params, cache_specific_params); diff --git a/libCacheSim/cache/eviction/S3FIFOd.c b/libCacheSim/cache/eviction/S3FIFOd.c index dad0e4330..c94679b53 100644 --- a/libCacheSim/cache/eviction/S3FIFOd.c +++ b/libCacheSim/cache/eviction/S3FIFOd.c @@ -152,7 +152,7 @@ cache_t *S3FIFOd_init(const common_cache_params_t ccache_params, } ccache_params_local.cache_size = ccache_params.cache_size / 10; - ccache_params_local.hashpower -= 4; + ccache_params_local.hashpower = MAX(4, ccache_params_local.hashpower - 4); params->small_eviction = FIFO_init(ccache_params_local, NULL); params->main_eviction = FIFO_init(ccache_params_local, NULL); snprintf(params->small_eviction->cache_name, CACHE_NAME_ARRAY_LEN, diff --git a/libCacheSim/cache/eviction/SLRUv0.c b/libCacheSim/cache/eviction/SLRUv0.c index 78c4c358e..d0e9e1ab0 100644 --- a/libCacheSim/cache/eviction/SLRUv0.c +++ b/libCacheSim/cache/eviction/SLRUv0.c @@ -93,7 +93,8 @@ cache_t *SLRUv0_init(const common_cache_params_t ccache_params, common_cache_params_t ccache_params_local = ccache_params; ccache_params_local.cache_size /= params->n_seg; - ccache_params_local.hashpower = MIN(16, ccache_params_local.hashpower - 4); + ccache_params_local.hashpower = + MAX(4, MIN(16, ccache_params_local.hashpower - 4)); params->LRUs[0] = LRU_init(ccache_params_local, NULL); for (int i = 1; i < params->n_seg; i++) { params->LRUs[i] = LRU_init(ccache_params_local, NULL); diff --git a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c index 4fd31d299..df792dd61 100644 --- a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c @@ -89,7 +89,7 @@ cache_t *LP_SFIFO_init(const common_cache_params_t ccache_params, } common_cache_params_t ccache_params_local = ccache_params; - ccache_params_local.hashpower -= 2; + ccache_params_local.hashpower = MAX(4, ccache_params_local.hashpower - 2); params->fifos = malloc(sizeof(cache_t *) * params->n_seg); for (int i = 0; i < params->n_seg; i++) { diff --git a/libCacheSim/cache/eviction/fifo/SFIFOv0.c b/libCacheSim/cache/eviction/fifo/SFIFOv0.c index 9a1a4f867..96ee1af84 100644 --- a/libCacheSim/cache/eviction/fifo/SFIFOv0.c +++ b/libCacheSim/cache/eviction/fifo/SFIFOv0.c @@ -101,7 +101,11 @@ cache_t *SFIFOv0_init(const common_cache_params_t ccache_params, common_cache_params_t ccache_params_local = ccache_params; ccache_params_local.cache_size /= params->n_queues; - ccache_params_local.hashpower /= MIN(16, ccache_params_local.hashpower - 4); + /* the divisor reaches zero once hashpower is 4 or less; guarded rather than + * rewritten, since dividing here (unlike the assignment SLRUv0 does) looks + * deliberate enough not to change behind the author's back */ + ccache_params_local.hashpower /= + MAX(1, MIN(16, ccache_params_local.hashpower - 4)); for (int i = 0; i < params->n_queues; i++) { params->FIFOs[i] = FIFO_init(ccache_params_local, NULL); } diff --git a/test/test_cli.sh b/test/test_cli.sh index c32ecdc12..6ec9cd717 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -240,6 +240,18 @@ for hp in 0 -1 40 99; do --num-req=20000 "--hashpower=${hp}" done +# Composite policies size their sub-caches by subtracting from this. Without a +# floor the result reached zero, which cache_struct_init reads as "unset" and +# replaces with the full-size default — so asking for a small table allocated +# several large ones instead. slruv0 at hashpower 4 took 18 MB against 6 MB at 5. +for algo in slruv0 s3fifod cacheus lru; do + for hp in 4 5 6 8; do + expect_ok "${algo} at --hashpower=${hp}" \ + "${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 10mb \ + --num-req=20000 "--hashpower=${hp}" + done +done + echo "running SLRU parameter validation tests" # n-seg divides the cache size and the reported percentages, and seg-size fills @@ -308,6 +320,16 @@ if [[ -x "${BIN_DIR}/mrcProfiler" ]]; then --size=0.1,0.5,10 done + # Rates outside (0, 1] are rejected. nan needs the negated comparison, since + # every ordinary comparison against it is false and it otherwise slipped + # through to produce an all-1.0 curve and a zero exit. + for rate in 0 -1 2 nan; do + expect_clean_error "mrcProfiler rejects sample rate ${rate}" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo=LRU --profiler=SHARDS --profiler-params="FIX_RATE,${rate},42" \ + --size=0.1,0.5,10 + done + # MINISIM looks its eviction algorithm up by name. It used to do that with # dlsym() against this executable, which cannot work when the constructors # sit in an unreferenced archive member, so every run aborted with From 2f1693427f5b431d724e264d53d3cc27b9cef6b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 03:01:34 +0000 Subject: [PATCH 18/33] fix: correct MINISIM's unsampled path and oracle-only policies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups from Codex, both reachable only because MINISIM now runs at all rather than aborting. Above a sample rate of 0.5 MINISIM stops sampling and replays the whole trace, but it still sized the miniature caches at mrc_size_vec[i] * sample_rate. Every request reached a cache smaller than the one the user asked for, so the curve was mislabelled: a 100MB point at rate 0.75 measured a 75MB cache, reporting 0.8257 where 100MB is 0.8225. The accounting further down already branched on whether a sampler exists; the sizing did not. Setting the effective rate to 1 on that path makes the two agree, and unsampled MINISIM now matches cachesim exactly — 0.822467, 0.766150 and 0.718500 against 0.8225, 0.7661 and 0.7185 at 100MB, 300MB and 500MB. belady and beladySize read next_access_vtime, which ordinary readers leave unset. cachesim refuses them on a non-oracle trace; MINISIM reached them through the registry without that check and produced a plausible-looking curve from garbage priorities, exiting 0. It now refuses them the same way, and still accepts them on oracleGeneral. testCLI gains the unsampled rates, an equality check against cachesim so the mislabelling cannot come back quietly, and both trace types for the two oracle-only policies. 170 checks, green plain and under LeakSanitizer. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/mrcProfiler/mrcProfiler.cpp | 20 +++++++++++++ test/test_cli.sh | 38 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index 4fdd71e39..de9838807 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -283,6 +283,11 @@ void mrcProfiler::MRCProfilerMINISIM::run() { sampler_t *sampler = nullptr; if (sample_rate > 0.5) { INFO("sample_rate is too large, do not sample\n"); + /* the whole trace is replayed, so the miniature caches have to be + * full-sized; leaving the requested rate in place would scale them down + * while every request still reached them, reporting the miss ratios of + * smaller caches than were asked for */ + sample_rate = 1.0; } else { sampler = create_spatial_sampler(sample_rate); set_spatial_sampler_salt(sampler, @@ -307,6 +312,21 @@ void mrcProfiler::MRCProfilerMINISIM::run() { reader_->init_params.sampler = sampler; reader_->sampler = sampler; + /* Belady and BeladySize read next_access_vtime, which ordinary readers leave + * at -2, so on any other trace they would produce a plausible-looking but + * meaningless curve rather than failing. cachesim checks this before building + * the cache; do the same here. */ + if (strcasecmp(params_.cache_algorithm_str, "belady") == 0 || + strcasecmp(params_.cache_algorithm_str, "beladySize") == 0) { + if (reader_->trace_type != ORACLE_GENERAL_TRACE && + reader_->trace_type != LCS_TRACE) { + ERROR( + "%s needs future information, so it only works on oracleGeneral and " + "lcs traces; convert with ./bin/traceConv\n", + params_.cache_algorithm_str); + } + } + // 3. run the simulate_with_multi_caches cache_t *caches[MAX_MRC_PROFILE_POINTS]; for (size_t i = 0; i < params_.profile_size.size(); i++) { diff --git a/test/test_cli.sh b/test/test_cli.sh index 6ec9cd717..831274f59 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -346,6 +346,44 @@ if [[ -x "${BIN_DIR}/mrcProfiler" ]]; then --algo=nosuchalgo --profiler=MINISIM --profiler-params=FIX_RATE,0.01,4 \ --size=0.1,0.5,10 + # Above 0.5 MINISIM stops sampling and replays the whole trace, so the + # miniature caches have to be full-sized. Scaling them by the requested rate + # reported the miss ratios of smaller caches than were asked for. + for rate in 0.6 0.75 1; do + expect_ok "mrcProfiler MINISIM unsampled at rate ${rate}" \ + "${BIN_DIR}/mrcProfiler" "${TRACE_ORACLE}" oracleGeneral \ + --algo=LRU --profiler=MINISIM --profiler-params="FIX_RATE,${rate},4" \ + --size=100MB,500MB,3 + done + + # With sampling off MINISIM replays everything, so it should agree with a + # straight cachesim run rather than approximate it. + # first row is the 100MB point; 104857600B + _minisim_unsampled=$("${BIN_DIR}/mrcProfiler" "${TRACE_ORACLE}" oracleGeneral \ + --algo=LRU --profiler=MINISIM --profiler-params=FIX_RATE,0.75,4 \ + --size=100MB,500MB,3 2>/dev/null | grep '^104857600B' | awk '{printf "%.4f", $2}') + _cachesim_exact=$("${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral lru 100mb \ + 2>/dev/null | tail -1 | grep -oE 'miss ratio [0-9.]+' | head -1 | awk '{printf "%.4f", $3}') + if [[ "${_minisim_unsampled}" == "${_cachesim_exact}" ]]; then + _report 0 "" + else + _report 1 "unsampled MINISIM (${_minisim_unsampled}) should match cachesim (${_cachesim_exact})" + fi + + # belady and beladySize read next_access_vtime, which ordinary readers leave + # unset, so on a non-oracle trace they must be refused rather than producing + # a plausible-looking curve. + for algo in belady beladySize; do + expect_clean_error "mrcProfiler MINISIM rejects ${algo} on a vscsi trace" \ + "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ + --algo="${algo}" --profiler=MINISIM --profiler-params=FIX_RATE,0.01,4 \ + --size=0.1,0.5,10 + expect_ok "mrcProfiler MINISIM accepts ${algo} on an oracle trace" \ + "${BIN_DIR}/mrcProfiler" "${TRACE_ORACLE}" oracleGeneral \ + --algo="${algo}" --profiler=MINISIM --profiler-params=FIX_RATE,0.01,4 \ + --size=0.1,0.5,10 + done + expect_ok "mrcProfiler SHARDS FIX_SIZE" \ "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ --algo=LRU --profiler=SHARDS --profiler-params=FIX_SIZE,2048,42 --size=100MB,1GB,10 From 610d644eb3e6c57cbd51f0bdf542dcd65c6f2379 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 03:14:44 +0000 Subject: [PATCH 19/33] fix: register the tinyLFU alias and correct the obj_id_is_num default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more from Codex. The API docs had the object-id default backwards. They said ids are hashed unless you opt into numeric parsing, but default_reader_init_params() sets obj_id_is_num to true — the opposite of what cachesim does, which overrides it to false. csv_cb1 checks that flag first and only hashes on the false path, so a library caller who took the defaults with string ids got strtoull() and a warning per row: row 1 -> obj_id=0 row 2 -> obj_id=0 row 3 -> obj_id=0 Six distinct objects collapsed into one, and the resulting miss ratio is meaningless rather than obviously wrong. Both doc/API.md and doc/advanced_lib.md now state the real default and that string ids need it set to false; with that, the same trace hashes correctly and repeated keys map to the same id. The default itself is a library behaviour question, so it is documented rather than changed here. cachesim accepts tinyLFU, but the registry did not, so MINISIM and any library caller got "cannot load ./libtinyLFU.so" — contradicting the registry's documented promise to accept the CLI's aliases. Added as a plain alias, and removed the special case that used to append window-size=0.01 when the caller had not given one: WTinyLFU's DEFAULT_PARAMS already sets exactly that before applying caller parameters, so it never changed anything. Verified identical between tinyLFU and wtinyLFU with no params, with main-cache=LRU, with window-size=0.2, and with both. testCLI covers tinyLFU through cachesim and MINISIM, and a csv of string ids asserting the miss ratio that four distinct objects give, so collapsing them to id 0 shows up as a failure rather than a plausible number. 174 checks. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/API.md | 8 +++++- doc/advanced_lib.md | 4 ++- libCacheSim/bin/cachesim/cache_init.h | 35 +++++++-------------------- libCacheSim/cache/cacheAlgoRegistry.c | 1 + test/test_cli.sh | 23 ++++++++++++++++-- 5 files changed, 41 insertions(+), 30 deletions(-) diff --git a/doc/API.md b/doc/API.md index 8f2112aa2..14530c5af 100644 --- a/doc/API.md +++ b/doc/API.md @@ -75,7 +75,13 @@ static inline reader_t *open_trace(const char *path, trace_type_e type, const reader_init_param_t *reader_init_param); ``` -Object ids are hashed unless you set `obj_id_is_num`, which you should do when the id field holds numbers. +> [!IMPORTANT] +> `default_reader_init_params()` sets `obj_id_is_num` to **true**, which is the opposite of what `cachesim` does. Set it to `false` yourself if the id field holds strings. The csv reader hashes string ids only on the `false` path; with `true` it runs them through `strtoull()`, which warns and yields `0`, so every object collapses into one and the miss ratio is meaningless. + +```c +reader_init_param_t p = default_reader_init_params(); +p.obj_id_is_num = false; /* string ids: hash them */ +``` ### Iterating over requests diff --git a/doc/advanced_lib.md b/doc/advanced_lib.md index 42f280a19..e8fcc3d00 100644 --- a/doc/advanced_lib.md +++ b/doc/advanced_lib.md @@ -113,7 +113,9 @@ open_trace(data_path, PLAIN_TXT_TRACE, NULL); ``` #### Setup a csv reader -The fields are 1-indexed and must match the trace. The sample `data/cloudPhysicsIO.csv` has the header `version,time,op,size,lbn`, so time is field 2, size is field 4, and the object id is field 5. Set `obj_id_is_num` when the id column holds numbers, otherwise the ids are hashed. +The fields are 1-indexed and must match the trace. The sample `data/cloudPhysicsIO.csv` has the header `version,time,op,size,lbn`, so time is field 2, size is field 4, and the object id is field 5. + +`obj_id_is_num` says whether the id column holds numbers. Note that `default_reader_init_params()` sets it to **true**, unlike `cachesim`, so a trace with string ids needs it set to `false` explicitly — otherwise the reader parses them with `strtoull()` and every id becomes `0` rather than being hashed. ```c reader_init_param_t init_params_csv = {.delimiter = ',', diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index 44af3694b..636c17897 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -44,35 +44,18 @@ static inline cache_t *create_cache(const char *trace_path, /* The name to constructor mapping lives in the library * (cache/cacheAlgoRegistry.c) so that the MINISIM profiler, which only knows * the algorithm by name, shares one table with the CLI. The cases below need - * more than a lookup — a smaller hash table, a default parameter, or a check - * that the trace carries the future information the algorithm needs — so - * they are handled here rather than in the registry. */ + * more than a lookup — a smaller hash table, or a check that the trace + * carries the future information the algorithm needs — so they are handled + * here rather than in the registry. + * + * tinyLFU used to be one of them, appending window-size=0.01 when the caller + * had not given one. WTinyLFU's DEFAULT_PARAMS already sets exactly that + * before applying the caller's parameters, so the append never changed + * anything; it is a plain alias in the registry now, which is also what makes + * it reachable from the MRC profiler. */ if (strcasecmp(eviction_algo, "hyperbolic") == 0) { cc_params.hashpower = MAX(cc_params.hashpower - 8, 16); cache = Hyperbolic_init(cc_params, eviction_params); - } else if (strcasecmp(eviction_algo, "tinyLFU") == 0) { - if (eviction_params == NULL || eviction_params[0] == '\0') { - cache = WTinyLFU_init(cc_params, NULL); - } else { - const char *window_size = strstr(eviction_params, "window-size="); - if (window_size == NULL) { - // Calculate exact size needed: original + ",window-size=0.01" + null - // terminator - size_t new_params_len = - strlen(eviction_params) + strlen(",window-size=0.01") + 1; - char *new_params = (char *)malloc(new_params_len); - if (new_params == NULL) { - ERROR("failed to allocate memory for new_params\n"); - abort(); - } - snprintf(new_params, new_params_len, "%s,window-size=0.01", - eviction_params); - cache = WTinyLFU_init(cc_params, new_params); - free(new_params); // Free the allocated memory - } else { - cache = WTinyLFU_init(cc_params, eviction_params); - } - } } else if (strcasecmp(eviction_algo, "belady") == 0) { if (strcasestr(trace_path, "oracleGeneral") == NULL && strcasestr(trace_path, "lcs") == NULL) { diff --git a/libCacheSim/cache/cacheAlgoRegistry.c b/libCacheSim/cache/cacheAlgoRegistry.c index 3c6c8b965..daa86e43d 100644 --- a/libCacheSim/cache/cacheAlgoRegistry.c +++ b/libCacheSim/cache/cacheAlgoRegistry.c @@ -69,6 +69,7 @@ static const cache_algo_entry_t g_cache_algos[] = { {"size", Size_init}, {"slru", SLRU_init}, {"slruv0", SLRUv0_init}, + {"tinyLFU", WTinyLFU_init}, {"twoq", TwoQ_init}, {"wtinyLFU", WTinyLFU_init}, /* these need future information and are only valid on oracle traces, so diff --git a/test/test_cli.sh b/test/test_cli.sh index 831274f59..24f223fc1 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -134,6 +134,23 @@ expect_clean_error "cachesim csv without obj-id-is-num" \ "${BIN_DIR}/cachesim" "${TRACE_CSV}" csv lru 1gb \ -t "time-col=2, obj-id-col=5, obj-size-col=4" +# String ids must be hashed, not run through strtoull. Getting this wrong +# collapses every object onto id 0, which shows up as an implausibly low miss +# ratio rather than an error. Four distinct objects in six requests, so a cache +# large enough to hold them all misses exactly four times. +cat >str-ids.csv <<'CSV' +time,id,size +1,alpha,100 +2,beta,200 +3,alpha,100 +4,gamma,300 +5,beta,200 +6,delta,400 +CSV +expect_output "cachesim hashes string object ids" "miss ratio 0\.6667" \ + "${BIN_DIR}/cachesim" str-ids.csv csv lru 1mb \ + -t "time-col=1,obj-id-col=2,obj-size-col=3,has-header=true" + echo "running -e print tests" # `-e print` runs before the cache is fully built, so the reporting path must @@ -159,7 +176,7 @@ ALL_ALGOS="2q 3LCache CAR GLCache RandomLRU arc arcv0 cacheus clock clock2qplus clockpro fifo fifo-merge fifo-reinsertion fifomerge flashProb gdsf gl-cache lecar lecarv0 lfu lfucpp lfuda lhd lirs lrb lru lru-k lru-prob nop pluginCache qdlp random randomTwo s3-fifo s3-fifov0 s3fifo s3fifod s3fifov0 - sieve size slru slruv0 twoq wtinyLFU" + sieve size slru slruv0 tinyLFU twoq wtinyLFU" n_skipped=0 for algo in ${ALL_ALGOS}; do @@ -334,7 +351,9 @@ if [[ -x "${BIN_DIR}/mrcProfiler" ]]; then # dlsym() against this executable, which cannot work when the constructors # sit in an unreferenced archive member, so every run aborted with # "undefined symbol: FIFO_init". Cover the non-LRU algorithms it exists for. - for algo in FIFO ARC S3FIFO sieve twoq clock lfu; do + # tinyLFU included: it is a cachesim alias, so name-based lookup has to + # accept it too, or the promise the registry documents is not kept. + for algo in FIFO ARC S3FIFO sieve twoq clock lfu tinyLFU; do expect_ok "mrcProfiler MINISIM with ${algo}" \ "${BIN_DIR}/mrcProfiler" "${TRACE}" vscsi \ --algo="${algo}" --profiler=MINISIM --profiler-params=FIX_RATE,0.01,4 \ From eb4634237199fc3248297a11c3d5d485d80c96e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 03:29:51 +0000 Subject: [PATCH 20/33] fix: widen the oracle trace check and make the doc images resolve Two review findings from the previous round. The MINISIM belady guard only accepted oracleGeneral and lcs, but the four oracleTwr/oracleTwrNS readers populate req->next_access_vtime too, so belady works on those and was being rejected. Move the list into a helper next to the reader formats that set the field, and name the actual trace type in the error so it says what was wrong rather than only what is allowed. quickstart_traceAnalyzer.md embeds its plots with raw tags pointing at /doc/plot/..., which resolve on github.com but became site-root paths in the built HTML, so all 18 images 404'd. Copy doc/plot and doc/assets into the output with html_extra_path and strip the prefix in the same source-read hook that already rewrites the Markdown links. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/conf.py | 17 ++++++++++++++-- libCacheSim/mrcProfiler/mrcProfiler.cpp | 27 ++++++++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 6e2ebf691..c0ca37e82 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -30,11 +30,17 @@ "_build", # Index for browsing the docs on GitHub; index.md is the Sphinx entry point. "README.md", - # Not documentation. + # Image directories, not pages. They are copied verbatim via + # html_extra_path below so the tags in the guides resolve. "plot", "assets", ] +# quickstart_traceAnalyzer.md embeds the plots with raw tags, so the +# files have to exist in the output. Copied rather than referenced, so the +# rendered docs do not depend on the repository being reachable. +html_extra_path = ["plot", "assets"] + # Generate anchors for headings so cross-file "#section" links resolve. myst_heading_anchors = 3 @@ -69,6 +75,12 @@ # Markdown inline links whose target leaves this directory. _LINK_RE = re.compile(r"\]\((/[^)\s]*|\.\./[^)\s]*)\)") +# The guides embed the trace-analysis plots with raw tags rather than +# Markdown, so those are not covered by _LINK_RE. html_extra_path copies the +# contents of doc/plot and doc/assets to the output root, so the site-root +# prefix has to come off for the images to resolve. +_IMG_RE = re.compile(r'(src=")/doc/(?:plot|assets)/([^"]+)"') + def _rewrite_target(match): target = match.group(1) @@ -85,7 +97,8 @@ def _rewrite_target(match): def _rewrite_repo_links(app, docname, source): - source[0] = _LINK_RE.sub(_rewrite_target, source[0]) + text = _IMG_RE.sub(r'\1\2"', source[0]) + source[0] = _LINK_RE.sub(_rewrite_target, text) def setup(app): diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index de9838807..b79695486 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -21,6 +21,23 @@ * rounds to this value and warns under -Wimplicit-const-int-float-conversion */ static constexpr double kHashSpaceSize = 18446744073709551616.0; +/* whether a reader of this type fills in req->next_access_vtime, which the + * Belady policies need. These are the formats whose readers set it; every other + * reader leaves it at -2. */ +static bool trace_type_has_next_access_vtime(trace_type_e trace_type) { + switch (trace_type) { + case ORACLE_GENERAL_TRACE: + case LCS_TRACE: + case ORACLE_SIM_TWR_TRACE: + case ORACLE_SYS_TWR_TRACE: + case ORACLE_SIM_TWRNS_TRACE: + case ORACLE_SYS_TWRNS_TRACE: + return true; + default: + return false; + } +} + mrcProfiler::MRCProfilerBase *mrcProfiler::create_mrc_profiler( mrc_profiler_e type, reader_t *reader, std::string output_path, const mrc_profiler_params_t ¶ms) { @@ -318,12 +335,12 @@ void mrcProfiler::MRCProfilerMINISIM::run() { * the cache; do the same here. */ if (strcasecmp(params_.cache_algorithm_str, "belady") == 0 || strcasecmp(params_.cache_algorithm_str, "beladySize") == 0) { - if (reader_->trace_type != ORACLE_GENERAL_TRACE && - reader_->trace_type != LCS_TRACE) { + if (!trace_type_has_next_access_vtime(reader_->trace_type)) { ERROR( - "%s needs future information, so it only works on oracleGeneral and " - "lcs traces; convert with ./bin/traceConv\n", - params_.cache_algorithm_str); + "%s needs future information, which %s traces do not carry; use an " + "oracle format such as oracleGeneral or lcs, or convert with " + "./bin/traceConv\n", + params_.cache_algorithm_str, g_trace_type_name[reader_->trace_type]); } } From 4466ab2a4ebb42831e3d3b69a23e0fcf668a6fa7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 15:54:43 +0000 Subject: [PATCH 21/33] fix: honour next_access_vtime in binary traces; scope the cwd note Two more review findings. The oracle-trace check classified readers by trace type alone, but the generic binary reader populates req->next_access_vtime whenever the caller points next_access_vtime_field at the right column, so a configured BIN_TRACE carries the future information Belady needs and was still being rejected. Ask the reader instead of assuming from the enum. No CLI exposes that field, but the profiler is part of the library, so a library caller can set it up. doc/index.md claimed every command in the docs runs from the build directory. That holds for the binaries, which is why the sample traces appear as ../data/, but not for the helper scripts: quickstart_traceAnalyzer.md invokes python3 scripts/traceAnalysis/*.py and debug.md invokes ./scripts/debug.sh, both relative to the repository root. Scope the sentence to what is true. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/index.md | 2 +- libCacheSim/mrcProfiler/mrcProfiler.cpp | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/doc/index.md b/doc/index.md index c113dca3d..acf411dd8 100644 --- a/doc/index.md +++ b/doc/index.md @@ -10,7 +10,7 @@ libCacheSim ships three things: New here? Start with [Install & Build](install.md), then [the cachesim guide](quickstart_cachesim.md). -The commands throughout these pages are run from the build directory (`_build/` if you followed the [README](https://github.com/1a1a11a/libCacheSim#build-and-install-libcachesim)), so the sample traces in `data/` are at `../data/`. +Commands that invoke a built binary — `./bin/cachesim`, `./bin/traceAnalyzer`, `./bin/mrcProfiler` — are run from the build directory (`_build/` if you followed the [README](https://github.com/1a1a11a/libCacheSim#build-and-install-libcachesim)), which is why the sample traces in `data/` appear as `../data/`. The helper scripts under `scripts/`, such as the plotting scripts and `debug.sh`, are run from the repository root instead, and those pages write the paths accordingly. ```{toctree} :maxdepth: 2 diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index b79695486..6a74a01e0 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -21,11 +21,17 @@ * rounds to this value and warns under -Wimplicit-const-int-float-conversion */ static constexpr double kHashSpaceSize = 18446744073709551616.0; -/* whether a reader of this type fills in req->next_access_vtime, which the - * Belady policies need. These are the formats whose readers set it; every other - * reader leaves it at -2. */ -static bool trace_type_has_next_access_vtime(trace_type_e trace_type) { - switch (trace_type) { +/* whether a reader fills in req->next_access_vtime, which the Belady policies + * need; every other reader leaves it at -2. + * + * Most oracle formats carry it unconditionally, so the trace type alone + * answers for them. The generic binary reader is the exception: it populates + * the field only when the caller points next_access_vtime_field at the right + * column, so a BIN_TRACE has to be asked rather than assumed. That is a + * library-only configuration today — no CLI exposes the field — but the + * profiler is part of the library, so a caller can set it up. */ +static bool reader_has_next_access_vtime(const reader_t *reader) { + switch (reader->trace_type) { case ORACLE_GENERAL_TRACE: case LCS_TRACE: case ORACLE_SIM_TWR_TRACE: @@ -33,6 +39,8 @@ static bool trace_type_has_next_access_vtime(trace_type_e trace_type) { case ORACLE_SIM_TWRNS_TRACE: case ORACLE_SYS_TWRNS_TRACE: return true; + case BIN_TRACE: + return reader->init_params.next_access_vtime_field > 0; default: return false; } @@ -335,7 +343,7 @@ void mrcProfiler::MRCProfilerMINISIM::run() { * the cache; do the same here. */ if (strcasecmp(params_.cache_algorithm_str, "belady") == 0 || strcasecmp(params_.cache_algorithm_str, "beladySize") == 0) { - if (!trace_type_has_next_access_vtime(reader_->trace_type)) { + if (!reader_has_next_access_vtime(reader_)) { ERROR( "%s needs future information, which %s traces do not carry; use an " "oracle format such as oracleGeneral or lcs, or convert with " From 2b0aab6a6f3e71947760b6d008c7d22835b617d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 16:16:04 +0000 Subject: [PATCH 22/33] Relicense to Apache-2.0; normalize binary next_access_vtime; restore plot Relicense from GPL-3.0 to Apache-2.0. LICENSE is now the canonical Apache text, byte-identical to apache.org, and the declarations that named GPL are updated to match: CITATION.cff, libCacheSim-node/package.json, the node README, CONTRIBUTING.md, and the README license section, which now names the license instead of only linking it. The vendored LHD code under libCacheSim/cache/eviction/LHD keeps its own MIT notice, which Apache-2.0 accommodates. The generic binary reader passed the "no next access" sentinel through raw, while the oracle readers normalize -1 to MAX_REUSE_DISTANCE. A binary trace carrying -1 therefore reached Belady, which rejects it outright. Normalize it in binary.c so the two readers agree: the same file read either way now produces an identical miss ratio curve. The popularity-decay section pointed at a plot that has never existed in the repo, which is why it was commented out; meanwhile the w92 plot generated for exactly that section sat unreferenced. Point the section at the real file and rewrite the caption, which had been copied from the object-size section and described two plots and the wrong quantity. All 18 images in the rendered docs now resolve. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- CITATION.cff | 2 +- CONTRIBUTING.md | 2 +- LICENSE | 876 ++++-------------- README.md | 2 +- doc/quickstart_traceAnalyzer.md | 11 +- libCacheSim-node/README.md | 2 +- libCacheSim-node/package.json | 2 +- .../traceReader/generalReader/binary.c | 8 + 8 files changed, 220 insertions(+), 685 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index e3f28b4b3..67f99cd19 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -20,7 +20,7 @@ keywords: - eviction algorithm - miss ratio curve - trace analysis -license: GPL-3.0-only +license: Apache-2.0 preferred-citation: type: conference-paper title: FIFO Queues Are All You Need for Cache Eviction diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0173db26c..b3ed3bc40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,4 +95,4 @@ Security issues should **not** be filed as public issues — see [SECURITY.md](S ## License -libCacheSim is [GPL-3.0](LICENSE) licensed. By contributing, you agree that your contributions are licensed under the same terms. +libCacheSim is [Apache-2.0](LICENSE) licensed. By contributing, you agree that your contributions are licensed under the same terms. diff --git a/LICENSE b/LICENSE index 9cecc1d46..d64569567 100644 --- a/LICENSE +++ b/LICENSE @@ -1,674 +1,202 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - {one line to give the program's name and a brief idea of what it does.} - Copyright (C) {year} {name of author} - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - {project} Copyright (C) {year} {fullname} - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index cdeb2de5d..7ce284250 100644 --- a/README.md +++ b/README.md @@ -477,7 +477,7 @@ GitHub's **Cite this repository** button uses [CITATION.cff](/CITATION.cff); [re ## License -See [LICENSE](LICENSE) for details. +libCacheSim is licensed under the [Apache License 2.0](LICENSE). ## Related * [PyMimircache](https://github.com/1a1a11a/PyMimircache): a python based cache trace analysis platform, now deprecated diff --git a/doc/quickstart_traceAnalyzer.md b/doc/quickstart_traceAnalyzer.md index 8c89ab19e..cdcf57d3b 100644 --- a/doc/quickstart_traceAnalyzer.md +++ b/doc/quickstart_traceAnalyzer.md @@ -246,16 +246,15 @@ There are two versions of the plots, one is line plot, and the other is a heatma python3 scripts/traceAnalysis/popularity_decay.py ${dataname}.popularityDecay_w300_obj ``` - +
## Advanced features ```bash diff --git a/libCacheSim-node/README.md b/libCacheSim-node/README.md index b1ce44aac..9a83eaabe 100644 --- a/libCacheSim-node/README.md +++ b/libCacheSim-node/README.md @@ -202,7 +202,7 @@ Contributions are welcome! Please see the main [libCacheSim repository](https:// ## License -GPL-3.0 - see the [LICENSE](https://github.com/1a1a11a/libCacheSim/blob/develop/LICENSE) file for details. This addon links libCacheSim statically, so the same terms apply to it. +Apache-2.0 - see the [LICENSE](https://github.com/1a1a11a/libCacheSim/blob/develop/LICENSE) file for details. This addon links libCacheSim statically, so the same terms apply to it. ## Related Projects diff --git a/libCacheSim-node/package.json b/libCacheSim-node/package.json index 6b4d50cdc..32232d64a 100644 --- a/libCacheSim-node/package.json +++ b/libCacheSim-node/package.json @@ -21,7 +21,7 @@ "libcachesim" ], "author": "Murphy Tian", - "license": "GPL-3.0-only", + "license": "Apache-2.0", "description": "Node.js bindings for libCacheSim - A high-performance cache simulator and analysis library supporting LRU, FIFO, S3-FIFO, Sieve and other caching algorithms", "repository": { "type": "git", diff --git a/libCacheSim/traceReader/generalReader/binary.c b/libCacheSim/traceReader/generalReader/binary.c index b198ce7ea..f6fe4c0b1 100644 --- a/libCacheSim/traceReader/generalReader/binary.c +++ b/libCacheSim/traceReader/generalReader/binary.c @@ -263,6 +263,14 @@ int binary_read_one_req(reader_t *reader, request_t *req) { if (params->next_access_vtime_field_idx > 0) { req->next_access_vtime = read_data(start + params->next_access_vtime_offset, params->next_access_vtime_format); + /* traces spell "no next access" as either -1 or INT64_MAX. The eviction + * algorithms expect MAX_REUSE_DISTANCE (which is INT64_MAX, so that form + * already arrives correct) and Belady rejects a raw -1 outright, so + * normalize it the way the oracle readers do and a binary trace behaves + * like an oracle one. */ + if (req->next_access_vtime == -1) { + req->next_access_vtime = MAX_REUSE_DISTANCE; + } } (reader->mmap_offset) += reader->item_size; From b68dcee1e545757b72774fe26d9b4722a053a3fc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 16:35:50 +0000 Subject: [PATCH 23/33] fix: shrink BeladySize's hash table in the MINISIM profiler BeladySize picks its victim by sampling the hash table, so cachesim shrinks the table by 8 before constructing it. MINISIM builds its miniature caches straight from the registry at a fixed hash power of 20, so it skipped that and gave every profile point a 1M-slot table. Peak RSS for an 8-point run drops from 32.0 MB to 10.2 MB. Applied in the profiler rather than in BeladySize_init. Moving it into the algorithm, the way Hyperbolic_init does, is arguably where it belongs and would fix every caller at once, but it overrides the hash power a library caller explicitly asks for: test_evictionAlgo builds BeladySize at hash power 20 and its recorded miss counts move (74329 -> 74347). That is a modelling change for the library API rather than a defect, so it is left for the maintainers; this commit changes only the profiler that had the problem. Hyperbolic needs no equivalent line because Hyperbolic_init already shrinks its own table, which is why it was unaffected. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/mrcProfiler/mrcProfiler.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index 6a74a01e0..04d429ec3 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -21,6 +21,9 @@ * rounds to this value and warns under -Wimplicit-const-int-float-conversion */ static constexpr double kHashSpaceSize = 18446744073709551616.0; +/* log2 of the hash table size for the miniature caches MINISIM simulates */ +static constexpr int kMiniSimHashPower = 20; + /* whether a reader fills in req->next_access_vtime, which the Belady policies * need; every other reader leaves it at -2. * @@ -352,13 +355,24 @@ void mrcProfiler::MRCProfilerMINISIM::run() { } } + /* BeladySize picks its victim by drawing samples from the hash table, so an + * oversized table costs memory and leaves the sampler probing empty buckets. + * cachesim shrinks it by 8 before constructing the cache; do the same here, + * since the miniature caches are built straight from the registry and would + * otherwise get a 1M-slot table each. Hyperbolic needs no such line because + * Hyperbolic_init already shrinks its own. */ + int minisim_hashpower = kMiniSimHashPower; + if (strcasecmp(params_.cache_algorithm_str, "beladySize") == 0) { + minisim_hashpower = MAX(minisim_hashpower - 8, 16); + } + // 3. run the simulate_with_multi_caches cache_t *caches[MAX_MRC_PROFILE_POINTS]; for (size_t i = 0; i < params_.profile_size.size(); i++) { size_t _cache_size = mrc_size_vec[i] * sample_rate; common_cache_params_t cc_params = {.cache_size = _cache_size, .default_ttl = 0, - .hashpower = 20, + .hashpower = minisim_hashpower, .consider_obj_metadata = false}; caches[i] = create_cache_using_plugin(params_.cache_algorithm_str, cc_params, nullptr); From d9462882402b34fc5b92e25175af12b8c5ed7bef Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 17:10:34 +0000 Subject: [PATCH 24/33] docs: warn that sampled beladySize MRCs are approximate BeladySize ranks candidates by next_access_vtime - cache->n_req. The first term counts requests in the full trace; the second counts only the requests the spatial sampler kept, so under sampling the reuse distance is inflated and the eviction order degrades. Belady is unaffected: it compares future times directly and never takes a difference. Measured on cloudPhysicsIO at a 100MB cache, error against the unsampled miss ratio: sample rate 0.5 beladySize 0.0126 belady 0.0003 lru 0.0023 sample rate 0.1 beladySize 0.0245 belady 0.0155 lru 0.0156 so at 0.5, where ordinary sampling error is still small, BeladySize is already an order of magnitude worse than either control. Warn and document rather than refuse or silently correct. Remapping future access times into sampled virtual time is a change to the sampler with its own accuracy trade-offs, which is the maintainers' call; what mattered here was that the profiler had been reporting the number without saying it was degraded. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/quickstart_mrcProfiler.md | 4 ++++ libCacheSim/mrcProfiler/mrcProfiler.cpp | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/doc/quickstart_mrcProfiler.md b/doc/quickstart_mrcProfiler.md index 1fc93b67e..821720eae 100644 --- a/doc/quickstart_mrcProfiler.md +++ b/doc/quickstart_mrcProfiler.md @@ -81,6 +81,10 @@ In the example below, `FIX_RATE,0.01,10` sets a `1%` sampling rate and `10` thre `--algo` accepts the same names as `cachesim`, so any built-in algorithm works — ARC, S3FIFO, sieve, twoq, and the rest. See the [README](/README.md#supported-algorithms) for the full list. +`belady` and `beladySize` need a trace that carries future access times, so they only run on an oracle format such as `oracleGeneral` or `lcs`; the profiler says so and stops otherwise. + +`beladySize` is additionally approximate under sampling, beyond the usual sampling error, and warns when you ask for it. It ranks candidates by reuse distance, computed as `next_access_vtime - n_req`, but `next_access_vtime` counts requests in the full trace while `n_req` counts only the requests the sampler kept, so the distance comes out inflated. On `cloudPhysicsIO` at a 100 MB cache, sample rate 0.5 puts it 0.0126 away from the unsampled miss ratio, against 0.0003 for `belady` and 0.0023 for LRU. Use `FIX_RATE,1,` for an exact run, or `belady`, which compares future times directly and is unaffected. + ### Ignoring Object Sizes To ignore object sizes (treat all objects as 1-byte): diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index 04d429ec3..4e1c77370 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -364,6 +364,26 @@ void mrcProfiler::MRCProfilerMINISIM::run() { int minisim_hashpower = kMiniSimHashPower; if (strcasecmp(params_.cache_algorithm_str, "beladySize") == 0) { minisim_hashpower = MAX(minisim_hashpower - 8, 16); + + /* BeladySize scores a candidate with next_access_vtime - cache->n_req. + * next_access_vtime counts requests in the full trace, but once the + * sampler drops requests, n_req counts only the ones that survived, so the + * two are in different units and the reuse distance comes out inflated. + * Belady is unaffected because it uses next_access_vtime as an ordering + * and never takes a difference. Measured on cloudPhysicsIO at a 100MB + * cache: at sample rate 0.5 BeladySize is off by 0.0126 against the + * unsampled miss ratio, where Belady is off by 0.0003 and LRU by 0.0023. + * Warn rather than refuse -- the curve is still in the right region, and + * remapping future times into sampled virtual time is a change to the + * sampler that belongs to the maintainers, not a silent correction here. */ + if (sampler != nullptr) { + WARN( + "beladySize scores candidates by reuse distance, which spatial " + "sampling distorts because next_access_vtime stays in full-trace " + "request numbers; the curve is approximate beyond the usual sampling " + "error. Use --profiler-params=FIX_RATE,1, for an exact " + "run, or belady, which is not affected.\n"); + } } // 3. run the simulate_with_multi_caches From 20cf194c9857d0c150c36916d8eb3cd19bb37f5a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 17:35:14 +0000 Subject: [PATCH 25/33] fix: charge each WTinyLFU sub-cache its own metadata, and match cachesim's hash power on exact MINISIM runs WTinyLFU_evict checked the main cache's capacity using cache->obj_md_size, which is the larger of the two sub-caches so that a caller asking the composite what it reserves is not told less than it really does. Against a FIFO or Clock main cache, which reserve nothing, that billed the window's 16 bytes and called the main cache full early; with an empty main cache it could reach to_evict() and dereference NULL. Each sub-cache is now charged its own. MINISIM built its miniature caches at hash power 20 even when sampling was disabled, while cachesim defaults to 24. Random, RandomTwo, RandomLRU and Hyperbolic draw eviction candidates through the hash mask, so the supposedly exact run drifted: randomTwo gave 0.8221 from cachesim and 0.8215 from MINISIM. Unsampled runs now use cachesim's hash power, including the extra reduction cachesim applies to hyperbolic and beladySize, and reproduce it exactly across all nine algorithms checked. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/cache/eviction/WTinyLFU.c | 7 ++++++- libCacheSim/mrcProfiler/mrcProfiler.cpp | 23 +++++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 38d8e7205..33059fff8 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -285,8 +285,13 @@ static void WTinyLFU_evict(cache_t *cache, const request_t *req) { /** only when main_cache is full, evict an obj from the main_cache **/ // if main_cache has enough space, insert the obj into main_cache + /* charge the main cache its own per-object overhead, not the composite's. + * cache->obj_md_size is the larger of the two sub-caches, so that a + * caller asking the composite what it reserves is not told less than it + * really does; using it here would bill a FIFO or Clock main cache for + * the window's 16 bytes and call it full early. */ if (main_cache->get_occupied_byte(main_cache) + - params->req_local->obj_size + cache->obj_md_size <= + params->req_local->obj_size + main_cache->obj_md_size <= main_cache->cache_size) { main_cache->insert(main_cache, params->req_local); diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index 4e1c77370..a5112be0d 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -21,9 +21,18 @@ * rounds to this value and warns under -Wimplicit-const-int-float-conversion */ static constexpr double kHashSpaceSize = 18446744073709551616.0; -/* log2 of the hash table size for the miniature caches MINISIM simulates */ +/* log2 of the hash table size for the miniature caches MINISIM simulates. The + * caches are small, so a smaller table than cachesim's is appropriate. */ static constexpr int kMiniSimHashPower = 20; +/* what cachesim uses, mirroring DEFAULT_HASHPOWER in bin/cachesim/cache_init.h. + * Above sample rate 0.5 MINISIM replays the whole trace, and that run is meant + * to be exact rather than approximate, so it has to size the table the way + * cachesim would: Random, RandomTwo, RandomLRU and Hyperbolic draw eviction + * candidates through the hash mask, so a different table gives a different + * curve. */ +static constexpr int kCacheSimHashPower = 24; + /* whether a reader fills in req->next_access_vtime, which the Belady policies * need; every other reader leaves it at -2. * @@ -359,9 +368,15 @@ void mrcProfiler::MRCProfilerMINISIM::run() { * oversized table costs memory and leaves the sampler probing empty buckets. * cachesim shrinks it by 8 before constructing the cache; do the same here, * since the miniature caches are built straight from the registry and would - * otherwise get a 1M-slot table each. Hyperbolic needs no such line because - * Hyperbolic_init already shrinks its own. */ - int minisim_hashpower = kMiniSimHashPower; + * otherwise get a 1M-slot table each. Hyperbolic gets it for a different + * reason: Hyperbolic_init shrinks its own table as well, so cachesim ends up + * two reductions down, and matching that is what makes an unsampled run + * reproduce cachesim rather than land 0.0001 away. */ + int minisim_hashpower = + (sampler == nullptr) ? kCacheSimHashPower : kMiniSimHashPower; + if (strcasecmp(params_.cache_algorithm_str, "hyperbolic") == 0) { + minisim_hashpower = MAX(minisim_hashpower - 8, 16); + } if (strcasecmp(params_.cache_algorithm_str, "beladySize") == 0) { minisim_hashpower = MAX(minisim_hashpower - 8, 16); From 5de5522a4864362ac41946d8cbdfab27228ed93f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 17:42:43 +0000 Subject: [PATCH 26/33] Address review findings across the split PRs plugin.c leaked the dlopen handle when dlsym failed. Nothing from the library is in use on that path, unlike the success path the existing comment covers, so it is closed rather than leaked. testCLI now sweeps hyperbolic, belady and beladySize, which cachesim special-cases and the registry also carries; the trace is oracleGeneral, so the Belady policies are valid on it. 174 checks becomes 180. mktemp -d is given an explicit template, since BSD mktemp rejects the bare form and the macOS job runs this target. The ctest entry gains SKIP_REGULAR_EXPRESSION, so a run that skipped itself for want of binaries or traces no longer reports as a pass. CONTRIBUTING.md described the CLI test by linking test/test_cli.sh. That file arrives in a different PR of this series, so the link would dangle for anyone merging the community files alone; the guidance now leads with what to cover and why the library tests will not catch it. adoption.md, which arrived on develop in #323, carries a BibTeX block whose % comment Pygments' bibtex lexer rejects. Harmless normally, but this branch sets fail_on_warning, so it broke the docs build; the comment moves into prose above the block. adoption.md is also added to the Sphinx toctree, which it needs to be reachable in the rendered docs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/adoption.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/adoption.md b/doc/adoption.md index 097a19696..83f868ebe 100644 --- a/doc/adoption.md +++ b/doc/adoption.md @@ -271,6 +271,8 @@ To cite the edition you read, pin it to a commit: open the file on GitHub and pr y, or run `git log -1 --format=%H -- doc/adoption.md` in a clone. Each edition's permalink is the commit that bumped its version in the changelog below. +Replace `` below with the permalink of the edition you read. + ```bibtex @techreport{libcachesim-adoption-census-2026, title = {libCacheSim Adoption Census}, @@ -279,7 +281,6 @@ permalink is the commit that bumped its version in the changelog below. number = {census v1.2.0}, year = {2026}, month = aug, - % replace with the permalink of the edition you read url = {https://github.com/1a1a11a/libCacheSim/blob//doc/adoption.md}, note = {Census date 2026-08-13} } From bb4466174b0f5883cc7d72f5f9759da58f54a474 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 17:52:10 +0000 Subject: [PATCH 27/33] Ship the license inside the npm package The addon declares Apache-2.0 and binding.gyp statically links vendor/liblibCacheSim.a, so every publication redistributes the library. The repository LICENSE sits one directory above the package, outside anything npm collects, so the tarball carried the declaration without the text: npm pack --dry-run -> 6 files, no LICENSE Apache-2.0 section 4(a) requires giving recipients a copy of the license, so add one to the package and list it in "files". The tarball is now 7 files with LICENSE among them. The gap predates the relicense -- the same tarball shipped while the package declared GPL-3.0 -- but relicensing is the moment to fix it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc (cherry picked from commit d739e5719354336cb2d32e005d1c9ced79a3bd42) --- libCacheSim-node/LICENSE | 202 ++++++++++++++++++++++++++++++++++ libCacheSim-node/package.json | 1 + 2 files changed, 203 insertions(+) create mode 100644 libCacheSim-node/LICENSE diff --git a/libCacheSim-node/LICENSE b/libCacheSim-node/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/libCacheSim-node/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/libCacheSim-node/package.json b/libCacheSim-node/package.json index 32232d64a..4d913f2aa 100644 --- a/libCacheSim-node/package.json +++ b/libCacheSim-node/package.json @@ -43,6 +43,7 @@ "prebuild": "^13.0.1" }, "files": [ + "LICENSE", "index.js", "cli.js", "binding.cc", From cc77b326f0a43d54acb8ff74bf047ead89c7d561 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 17:58:15 +0000 Subject: [PATCH 28/33] Harden the CLI test helper, and fix the documented csv reader setup _expect_clean_failure accepted any non-zero exit whose output contained the word "error". A sanitizer turns a crash into exit 1 with "ERROR: AddressSanitizer", so under the sanitizer builds this suite is meant to run in, a crash regression would have been counted as a clean rejection. Sanitizer signatures are now checked first. SKIP_REGULAR_EXPRESSION arrived in CMake 3.16 and this project declares 3.12, where setting it is an error, so it is now guarded by a version check. The documented csv reader used a bare designated initializer, which zeroes every field it does not name -- cap_at_n_req, block_size and ignore_size_zero_req all default to something other than zero. It also set has_header without has_header_set, and the reader auto-detects unless the flag is raised, so the assignment did nothing; it happened to work only because detection guessed right on this file. The snippet now starts from default_reader_init_params() and raises both _set flags. Compiled and run: it reads 113872 requests with first id 42932745 size 512, matching the vscsi reader exactly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- doc/advanced_lib.md | 17 +++++++++++------ test/CMakeLists.txt | 8 ++++++-- test/test_cli.sh | 6 ++++++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/doc/advanced_lib.md b/doc/advanced_lib.md index e8fcc3d00..8efa4aeaf 100644 --- a/doc/advanced_lib.md +++ b/doc/advanced_lib.md @@ -117,13 +117,18 @@ The fields are 1-indexed and must match the trace. The sample `data/cloudPhysics `obj_id_is_num` says whether the id column holds numbers. Note that `default_reader_init_params()` sets it to **true**, unlike `cachesim`, so a trace with string ids needs it set to `false` explicitly — otherwise the reader parses them with `strtoull()` and every id becomes `0` rather than being hashed. +Start from `default_reader_init_params()` rather than a bare designated initializer: the defaults for `cap_at_n_req`, `block_size` and `ignore_size_zero_req` are not zero, and a struct literal would silently set them to zero. `has_header` and `obj_id_is_num` are each paired with a `_set` flag; the reader auto-detects unless you raise the flag, so assigning the value alone has no effect. + ```c -reader_init_param_t init_params_csv = {.delimiter = ',', - .time_field = 2, - .obj_size_field = 4, - .obj_id_field = 5, - .obj_id_is_num = true, - .has_header = true}; +reader_init_param_t init_params_csv = default_reader_init_params(); +init_params_csv.delimiter = ','; +init_params_csv.time_field = 2; +init_params_csv.obj_size_field = 4; +init_params_csv.obj_id_field = 5; +init_params_csv.obj_id_is_num = true; +init_params_csv.obj_id_is_num_set = true; +init_params_csv.has_header = true; +init_params_csv.has_header_set = true; reader_t *reader_csv = open_trace("data/cloudPhysicsIO.csv", CSV_TRACE, &init_params_csv); ``` diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 317d79bfe..196ad7ad3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -68,8 +68,12 @@ add_test(NAME testCLI WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) # The script prints SKIP: and exits 0 when the binaries or sample traces are # absent. Without this, ctest reports that as a pass, so a run that tested -# nothing looks the same as a run that tested everything. -set_tests_properties(testCLI PROPERTIES SKIP_REGULAR_EXPRESSION "^SKIP:") +# nothing looks the same as a run that tested everything. The property arrived +# in CMake 3.16 and this project supports 3.12, where the run still passes +# rather than skipping -- no worse than before, just not as clear. +if(NOT CMAKE_VERSION VERSION_LESS "3.16") + set_tests_properties(testCLI PROPERTIES SKIP_REGULAR_EXPRESSION "^SKIP:") +endif() # add_test(NAME testMrcProfiler COMMAND testMrcProfiler WORKING_DIRECTORY .) # if (ENABLE_GLCACHE) diff --git a/test/test_cli.sh b/test/test_cli.sh index b2f98b703..73bcd3ea1 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -109,6 +109,12 @@ expect_clean_error() { _report 1 "${desc} crashed with signal (exit ${rc})" elif [[ ${rc} -eq 0 ]]; then _report 1 "${desc} was accepted but should have been rejected" + elif grep -qE "(AddressSanitizer|LeakSanitizer|ThreadSanitizer|MemorySanitizer|UndefinedBehaviorSanitizer|runtime error:)" <<<"${out}"; then + # a sanitizer turns a crash into exit 1 with "ERROR: AddressSanitizer", + # which the generic check below would read as a clean rejection. Test + # this first so a crash cannot pass merely by printing the word error. + _report 1 "${desc} tripped a sanitizer (exit ${rc})" + grep -E "(Sanitizer|runtime error:)" <<<"${out}" | head -2 | sed 's/^/ /' elif ! grep -qi "error" <<<"${out}"; then _report 1 "${desc} failed without an error message (exit ${rc})" else From b5b9472a17e3f941391b91817eecb5601c4dc229 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 18:11:36 +0000 Subject: [PATCH 29/33] fix: size MINISIM's caches by the sample rate actually in force create_spatial_sampler stores sampling_ratio_inv as an integer, so it can only represent rates of the form 1/n: it truncates 1/0.3 to 3 and keeps a third of objects. The cache sizing used the rate that was asked for rather than the one in force, making every cache too small by that ratio -- 10% at 0.3 -- while the curve was still reported against the requested size, so the error did not show up anywhere in the output. Demonstrable by asking for two rates that quantize to the same sampler: 0.3 and 0.3333333 both give sampling_ratio_inv 3 and therefore sample identical objects, so any difference between their curves is the bug and nothing else. before rate=0.3333333 0.8193 0.7637 rate=0.3 0.8204 0.7788 after rate=0.3333333 0.8193 0.7637 rate=0.3 0.8193 0.7637 Rates that are already of the form 1/n, including the 0.5 boundary and the 0.01 used throughout the docs, are unaffected. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/mrcProfiler/mrcProfiler.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libCacheSim/mrcProfiler/mrcProfiler.cpp b/libCacheSim/mrcProfiler/mrcProfiler.cpp index a5112be0d..bdfe29c7a 100644 --- a/libCacheSim/mrcProfiler/mrcProfiler.cpp +++ b/libCacheSim/mrcProfiler/mrcProfiler.cpp @@ -329,6 +329,20 @@ void mrcProfiler::MRCProfilerMINISIM::run() { sampler = create_spatial_sampler(sample_rate); set_spatial_sampler_salt(sampler, 10000019); // TODO: salt can be changed by params + + /* the sampler keeps one object in sampling_ratio_inv, an integer, so it can + * only represent rates of the form 1/n: create_spatial_sampler truncates + * 1/0.3 to 3 and then keeps a third. Sizing the caches by the rate that was + * asked for rather than the one in force makes them too small by that + * ratio -- 10% at 0.3 -- and the curve is reported against the size that + * was asked for, so the error is invisible in the output. Take the rate the + * sampler actually applies. */ + double effective_rate = 1.0 / sampler->sampling_ratio_inv; + if (effective_rate != sample_rate) { + INFO("sample rate %.6f is not of the form 1/n, using %.6f\n", sample_rate, + effective_rate); + sample_rate = effective_rate; + } } // 1. obtain the n_req_, sum_obj_size_req, sampled_cnt and sampled_size From e2a30b22a4cf1100145b849da3f3278202e8e392 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 18:18:37 +0000 Subject: [PATCH 30/33] fix: charge WTinyLFU's parent capacity check the window's metadata cache_get_base()'s eviction loop charges an incoming object cache->obj_md_size before deciding whether to evict. Every incoming object is inserted into the window, so that field has to be the window's overhead; setting it to the maximum of the two sub-caches made the loop reserve up to 40 bytes for a 16-byte window insertion whenever the main policy was ARC, LeCaR or Cacheus, evicting earlier than the configuration calls for. All three sites now charge the cache the object is actually entering: this one and WTinyLFU_can_insert() the window, WTinyLFU_evict() the main cache on promotion. Measured on cloudPhysicsIO at 100MB with metadata on, the effect is small -- LeCaR moves 0.7894 to 0.7895 and ARC, LRU, SLRU and FIFO do not move at all, because objects are large relative to the difference. Correct rather than consequential, but it also removes the last place where the composite charged an overhead belonging to a cache the object was not entering. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/cache/eviction/WTinyLFU.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libCacheSim/cache/eviction/WTinyLFU.c b/libCacheSim/cache/eviction/WTinyLFU.c index 33059fff8..752ab3a1d 100644 --- a/libCacheSim/cache/eviction/WTinyLFU.c +++ b/libCacheSim/cache/eviction/WTinyLFU.c @@ -149,8 +149,15 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, * cache_can_insert_default() uses, so take the larger of the two: an object * that does not fit under the heavier policy does not fit in this cache. * WTinyLFU_can_insert() checks each sub-cache against its own overhead. */ - cache->obj_md_size = - MAX(params->LRU->obj_md_size, params->main_cache->obj_md_size); + /* Every incoming object is inserted into the window, and this field is + * what cache_get_base()'s capacity loop charges an incoming object, so it + * is the window's overhead rather than the pair's maximum. The other two + * sites each charge the cache the object is actually entering: + * WTinyLFU_can_insert() the window, WTinyLFU_evict() the main cache on + * promotion. Using the maximum here made the loop reserve up to 40 bytes + * for a 16-byte window insertion with an ARC, LeCaR or Cacheus main + * cache. */ + cache->obj_md_size = params->LRU->obj_md_size; } snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "WTinyLFU-w%.2lf-%s", From 8c5b169be82452bc397f91bba96d5f1bab5d313d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 13 Aug 2026 18:28:15 +0000 Subject: [PATCH 31/33] test: only skip algorithms that a build flag can actually remove The sweeps skipped any algorithm whose run reported "do not support algorithm", on the assumption that the message means an optional feature was not compiled in. cache_init.h emits the same message when a name is missing from the registry for any reason, so a mandatory algorithm silently dropping out of g_cache_algos would have been skipped rather than reported -- the exact regression the registry refactor could introduce, and the one this sweep is here to catch. Only 3LCache, GLCache, gl-cache and lrb sit behind build flags, so the skip is restricted to those and everything else fails. Verified by deleting an entry and rebuilding. gdsf is covered by the sweeps and nothing else, so before this it disappeared without a sound: FAIL: gdsf -e print (exit 134) do not support algorithm gdsf FAIL: gdsf replay (exit 134) do not support algorithm gdsf (4 algorithms not compiled in, skipped) The four genuinely optional ones still skip. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- test/test_cli.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/test_cli.sh b/test/test_cli.sh index 73bcd3ea1..f5355f51a 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -176,8 +176,15 @@ expect_output "s3fifod -e print" "fifo-size-ratio=" \ # Sweep every algorithm the CLI registers rather than a hand-picked list: the # crashes this covers were found in variants that a shorter list missed -# (s3fifov0, flashProb). Algorithms behind an optional build flag report -# "do not support algorithm" and are skipped. +# (s3fifov0, flashProb). +# +# Only these four sit behind a build flag (ENABLE_3L_CACHE, ENABLE_GLCACHE, +# ENABLE_LRB) and may legitimately be absent. Skipping on the "do not support +# algorithm" message alone would also skip a mandatory algorithm that had +# silently dropped out of the registry, which is precisely the regression this +# sweep exists to catch. +OPTIONAL_ALGOS=" 3LCache GLCache gl-cache lrb " + ALL_ALGOS="2q 3LCache CAR GLCache RandomLRU arc arcv0 cacheus clock clock2qplus clockpro fifo fifo-merge fifo-reinsertion fifomerge flashProb gdsf gl-cache lecar lecarv0 lfu lfucpp lfuda lhd lirs lrb lru lru-k lru-prob nop @@ -189,7 +196,8 @@ n_skipped=0 for algo in ${ALL_ALGOS}; do out=$("${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 1gb -e print 2>&1) rc=$? - if [[ ${rc} -ne 0 ]] && grep -qi "do not support algorithm" <<<"${out}"; then + if [[ ${rc} -ne 0 ]] && grep -qi "do not support algorithm" <<<"${out}" && + [[ ${OPTIONAL_ALGOS} == *" ${algo} "* ]]; then n_skipped=$((n_skipped + 1)) continue fi @@ -219,7 +227,8 @@ for algo in ${ALL_ALGOS}; do out=$("${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral "${algo}" 10mb \ --num-req=20000 2>&1) rc=$? - if [[ ${rc} -ne 0 ]] && grep -qi "do not support algorithm" <<<"${out}"; then + if [[ ${rc} -ne 0 ]] && grep -qi "do not support algorithm" <<<"${out}" && + [[ ${OPTIONAL_ALGOS} == *" ${algo} "* ]]; then n_skipped=$((n_skipped + 1)) continue fi From 9658f2929048874d048e5e5eead6f8c826e97c4b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 20:03:57 +0000 Subject: [PATCH 32/33] fix: free MQ's parameter string on the -e print early exit MultiQueue arrived on develop in #318 with the same leak this branch fixed in 29 other files: MQ_parse_params strdup's the parameter string and frees it at the end, but the "print" branch calls exit(0) first. LeakSanitizer reports 6 bytes, the length of "print". Found by CI rather than by reading: merging develop added mq and multiqueue to the registry, the testCLI sweep covers every registered name, and the ubuntu job runs it under LeakSanitizer: FAIL: mq -e print (exit 23) SUMMARY: LeakSanitizer: 6 byte(s) leaked in 1 allocation(s). FAIL: multiqueue -e print (exit 23) which is the sweep doing exactly what it was added for -- a brand-new algorithm inherited the defect the same day it landed, and nothing else in the suite would have looked at it. Reproduced and confirmed fixed in a local -fsanitize=leak build with CI's ASAN_OPTIONS: ctest 10/10, testCLI 184 checks. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- libCacheSim/cache/eviction/MQ.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libCacheSim/cache/eviction/MQ.c b/libCacheSim/cache/eviction/MQ.c index 727f2ac9f..32e3a6748 100644 --- a/libCacheSim/cache/eviction/MQ.c +++ b/libCacheSim/cache/eviction/MQ.c @@ -505,6 +505,7 @@ static void MQ_parse_params(cache_t *cache, const char *cache_specific_params) { } } else if (strcasecmp(key, "print") == 0) { printf("parameters: %s\n", MQ_current_params(params)); + free(old_params_str); exit(0); } else { ERROR("%s does not have parameter %s\n", cache->cache_name, key); From 01bafc429e4b0ce175e5c69076c1c877eaf63cbe Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 20:15:45 +0000 Subject: [PATCH 33/33] fix: keep the "use the default" hash power sentinel intact cache_struct_init() reads a non-positive hashpower as "caller did not choose", substituting HASH_POWER_DEFAULT. The clamps added for --hashpower ran before that check, so MAX(4, 0 - 4) turned the sentinel into a literal hash power of 4 -- a 16-bucket table -- for any library caller using a designated initializer without setting the field. The reduction now applies only to an explicitly positive value, in Cacheus, S3FIFOd, SLRUv0 and LP_SFIFO alike. Worth recording that the predicted consequence was backwards: restoring the sentinel makes the sample replay *slower*, 54ms to 301ms, because HASH_POWER_DEFAULT allocates 8M buckets per sub-cache while a 16-bucket table simply grows. The point stands anyway -- 301ms is what the library did before this branch, and silently redefining an unset hashpower is not a change to make in passing. The bug the clamps were added for is still fixed: cachesim slruv0 memory stays monotonic across --hashpower 4, 5, 6, 8 and 24, with no inversion. Also: FAQ.md described next_access_vtime as the number of requests until the next access. It is the absolute 1-based request index of that access, and algorithms subtract the current request count themselves, so a trace built to the documented meaning would evict in the wrong order. Verified against the sample: request 7 stores 19, and that object is next requested at request 19. And testCLI's unsampled-MINISIM equality check compared two extracted strings without requiring either to be non-empty, so if both extractions stopped matching it would have passed on "" == "" -- the same shape as the skip logic fixed earlier today, an assertion that can succeed without observing anything. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YUq1vM4g82TkaX2jvLmuQc --- FAQ.md | 2 +- libCacheSim/cache/eviction/Cacheus.c | 7 ++++++- libCacheSim/cache/eviction/S3FIFOd.c | 6 +++++- libCacheSim/cache/eviction/SLRUv0.c | 8 ++++++-- libCacheSim/cache/eviction/fifo/LP_SFIFO.c | 6 +++++- test/test_cli.sh | 7 ++++++- 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/FAQ.md b/FAQ.md index b50461f95..fdeaa8af8 100644 --- a/FAQ.md +++ b/FAQ.md @@ -30,7 +30,7 @@ oracleGeneral traces are usually stored zstd-compressed, and libCacheSim reads t In the sample [cloudPhysicsIO.csv](/data/cloudPhysicsIO.csv), time is in seconds and object size is in bytes. -`next_access_vtime` is a *logical* time: the number of requests between the current request and the next request to the same object, or `-1` when the object is never accessed again. Algorithms that need future information, such as [Belady](/libCacheSim/cache/eviction/Belady.c) and BeladySize, rely on it, which is why they only work on oracle traces. +`next_access_vtime` is a *logical* time: the 1-based request index at which this object is next requested — an absolute position in the trace, not the distance to it — or `-1` when the object is never accessed again. Algorithms subtract the current request count themselves, so encoding a distance here silently changes eviction order. In `cloudPhysicsIO.oracleGeneral.bin`, for instance, request 7 stores `19` and that object is next seen at request 19. Algorithms that need future information, such as [Belady](/libCacheSim/cache/eviction/Belady.c) and BeladySize, rely on it, which is why they only work on oracle traces. Object ids are hashed unless the reader is told they are already numeric. Pass `obj-id-is-num=true` in `--trace-type-params` when the id column holds numbers — `cachesim` stops with an error if you leave it out on such a trace. diff --git a/libCacheSim/cache/eviction/Cacheus.c b/libCacheSim/cache/eviction/Cacheus.c index 50d0ba47a..541fc6fb4 100644 --- a/libCacheSim/cache/eviction/Cacheus.c +++ b/libCacheSim/cache/eviction/Cacheus.c @@ -72,7 +72,12 @@ cache_t *Cacheus_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { common_cache_params_t updated_cc_params = ccache_params; /* reduce the hash table size */ - updated_cc_params.hashpower = MAX(4, updated_cc_params.hashpower - 2); + /* only shrink an explicitly requested hash power: cache_struct_init reads + * a non-positive value as "use the default", and clamping would turn that + * sentinel into a 16-bucket table. */ + if (updated_cc_params.hashpower > 0) { + updated_cc_params.hashpower = MAX(4, updated_cc_params.hashpower - 2); + } cache_t *cache = cache_struct_init("Cacheus", updated_cc_params, cache_specific_params); diff --git a/libCacheSim/cache/eviction/S3FIFOd.c b/libCacheSim/cache/eviction/S3FIFOd.c index c94679b53..3c2e63641 100644 --- a/libCacheSim/cache/eviction/S3FIFOd.c +++ b/libCacheSim/cache/eviction/S3FIFOd.c @@ -152,7 +152,11 @@ cache_t *S3FIFOd_init(const common_cache_params_t ccache_params, } ccache_params_local.cache_size = ccache_params.cache_size / 10; - ccache_params_local.hashpower = MAX(4, ccache_params_local.hashpower - 4); + /* see Cacheus_init: a non-positive hash power is the "use the default" + * sentinel and must survive untouched. */ + if (ccache_params_local.hashpower > 0) { + ccache_params_local.hashpower = MAX(4, ccache_params_local.hashpower - 4); + } params->small_eviction = FIFO_init(ccache_params_local, NULL); params->main_eviction = FIFO_init(ccache_params_local, NULL); snprintf(params->small_eviction->cache_name, CACHE_NAME_ARRAY_LEN, diff --git a/libCacheSim/cache/eviction/SLRUv0.c b/libCacheSim/cache/eviction/SLRUv0.c index d0e9e1ab0..aad375fa2 100644 --- a/libCacheSim/cache/eviction/SLRUv0.c +++ b/libCacheSim/cache/eviction/SLRUv0.c @@ -93,8 +93,12 @@ cache_t *SLRUv0_init(const common_cache_params_t ccache_params, common_cache_params_t ccache_params_local = ccache_params; ccache_params_local.cache_size /= params->n_seg; - ccache_params_local.hashpower = - MAX(4, MIN(16, ccache_params_local.hashpower - 4)); + /* see Cacheus_init: a non-positive hash power is the "use the default" + * sentinel and must survive untouched. */ + if (ccache_params_local.hashpower > 0) { + ccache_params_local.hashpower = + MAX(4, MIN(16, ccache_params_local.hashpower - 4)); + } params->LRUs[0] = LRU_init(ccache_params_local, NULL); for (int i = 1; i < params->n_seg; i++) { params->LRUs[i] = LRU_init(ccache_params_local, NULL); diff --git a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c index df792dd61..075feeba2 100644 --- a/libCacheSim/cache/eviction/fifo/LP_SFIFO.c +++ b/libCacheSim/cache/eviction/fifo/LP_SFIFO.c @@ -89,7 +89,11 @@ cache_t *LP_SFIFO_init(const common_cache_params_t ccache_params, } common_cache_params_t ccache_params_local = ccache_params; - ccache_params_local.hashpower = MAX(4, ccache_params_local.hashpower - 2); + /* see Cacheus_init: a non-positive hash power is the "use the default" + * sentinel and must survive untouched. */ + if (ccache_params_local.hashpower > 0) { + ccache_params_local.hashpower = MAX(4, ccache_params_local.hashpower - 2); + } params->fifos = malloc(sizeof(cache_t *) * params->n_seg); for (int i = 0; i < params->n_seg; i++) { diff --git a/test/test_cli.sh b/test/test_cli.sh index 9e1d88778..212011844 100755 --- a/test/test_cli.sh +++ b/test/test_cli.sh @@ -400,7 +400,12 @@ if [[ -x "${BIN_DIR}/mrcProfiler" ]]; then --size=100MB,500MB,3 2>/dev/null | grep '^104857600B' | awk '{printf "%.4f", $2}') _cachesim_exact=$("${BIN_DIR}/cachesim" "${TRACE_ORACLE}" oracleGeneral lru 100mb \ 2>/dev/null | tail -1 | grep -oE 'miss ratio [0-9.]+' | head -1 | awk '{printf "%.4f", $3}') - if [[ "${_minisim_unsampled}" == "${_cachesim_exact}" ]]; then + # both sides must actually have been extracted: if the output formats change + # and neither grep matches, "" == "" would record this assertion as passed + # without either miss ratio having been observed. + if [[ -z "${_minisim_unsampled}" || -z "${_cachesim_exact}" ]]; then + _report 1 "could not read a miss ratio to compare (minisim='${_minisim_unsampled}' cachesim='${_cachesim_exact}')" + elif [[ "${_minisim_unsampled}" == "${_cachesim_exact}" ]]; then _report 0 "" else _report 1 "unsampled MINISIM (${_minisim_unsampled}) should match cachesim (${_cachesim_exact})"