From 976bb79b14a3af74ee7bc8e24e19c669af9d249e Mon Sep 17 00:00:00 2001 From: "Kevin A. Brown" Date: Mon, 17 Jun 2024 00:13:12 -0500 Subject: [PATCH 01/13] MPI Replay: remove print_surrogate_stats() to compile cleanly --- src/network-workloads/model-net-mpi-replay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network-workloads/model-net-mpi-replay.c b/src/network-workloads/model-net-mpi-replay.c index 1433b2a3..642b0bcc 100644 --- a/src/network-workloads/model-net-mpi-replay.c +++ b/src/network-workloads/model-net-mpi-replay.c @@ -3756,7 +3756,7 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv ) if(alloc_spec) codes_jobmap_destroy(jobmap_ctx); - print_surrogate_stats(); + //print_surrogate_stats(); #ifdef USE_RDAMARIS } // end if(g_st_ross_rank) From 7f42f4aa5f79a872004b8c11a5bbda998a9ac6c3 Mon Sep 17 00:00:00 2001 From: "Kevin A. Brown" Date: Thu, 4 Jul 2024 23:17:00 -0500 Subject: [PATCH 02/13] director-b: started adding director LP for mpi-replay --- src/network-workloads/model-net-mpi-replay.c | 426 ++++++++++++++++++- 1 file changed, 411 insertions(+), 15 deletions(-) diff --git a/src/network-workloads/model-net-mpi-replay.c b/src/network-workloads/model-net-mpi-replay.c index 642b0bcc..eb1dc685 100644 --- a/src/network-workloads/model-net-mpi-replay.c +++ b/src/network-workloads/model-net-mpi-replay.c @@ -19,6 +19,288 @@ #include "codes/codes-jobmap.h" #include "codes/congestion-controller-core.h" + + +/* ========================================================== + START OF Director Code (To be moved to separate files) + ========================================================== +*/ + +struct +{ + int surr_iter_start; + int surr_iter_end; +} director_config_global; + +//struct director_config_struct dir_config_global; + + +#define DIR_MAX_PREDICTION 10 +#define NUM_DIR_TO_NW_EVENT 20 + +enum SIMULATION_MODE +{ + SIM_MODE_PDES=1, + SIM_MODE_ITERATION_SURROGATE, +}; + +typedef struct director_state director_state; +typedef struct director_message director_message; +typedef struct director_annotation director_annotation; + +enum DIR_EVENTS +{ + DIR_AN_ITER_MARK=1, + DIR_OP_NW, + DIR_REGISTERED_EVENT__SWITCH_TO_SURR, + DIR_REGISTERED_EVENT__SWITCH_TO_PDES, + DIR_REGISTERED_EVENT__MOVE_TO_NEXT, +}; + +enum DIR_OPERATIONS //currently unused +{ + DIR_AN_WK_START=1, + DIR_AN_WK_ITERATION_END, + DIR_AN_WK_END, + DIR_OP_SEND, + DIR_OP_RECV, +}; + +// state of the director LP +struct director_state +{ + tw_lpid director_id; + int simulation_mode; + + tw_stime predictions[DIR_MAX_PREDICTION]; + + void *nw_event_ptr[NUM_DIR_TO_NW_EVENT]; + int nw_event_size[NUM_DIR_TO_NW_EVENT]; + + tw_lpid nw_lpid; +}; + +// director event message struct +struct director_message +{ + int msg_type; + int op_type; + int num_rngs; + int value; + model_net_event_return event_rc; + struct codes_workload_op * mpi_op; + + void *buffer; // this pointer MUST be at the end of the structure +}; + +// director annotation struct +struct director_annotation +{ + int an_type; + int an_value; +}; + +/* Trigger Director Event from within network model*/ +static void codes_issue_director_event(tw_lp* lp, tw_lpid director_lpid, int dir_event_type, int value) +{ + + tw_event *e; + struct director_message* msg; + + tw_stime ts; + + ts = .0001; // Todo: maybe this should be dynamically adjustable + + e = tw_event_new(director_lpid, ts, lp ); + msg = (director_message*)tw_event_data(e); + + msg->msg_type = dir_event_type; + msg->value = value; + tw_event_send(e); +} + +/* Trigger CODES Event From Director */ +static void director_issue_codes_event(director_state * s, tw_lpid nw_lpid, int dir_registered_event_type, tw_stime ts, tw_lp* lp) +{ + + tw_event *e; + void* msg; + + //printf("==DIR: ts: %lf\n", ts); + e = tw_event_new(nw_lpid, ts, lp); + msg = (void*)tw_event_data(e); + + memcpy(msg, s->nw_event_ptr[dir_registered_event_type], s->nw_event_size[dir_registered_event_type]); + + //msg->msg_type = dir_registered_event_type; + tw_event_send(e); +} + +void director_register_events(director_state * s, director_message * msg, tw_lp * lp) +{ + int dir_registered_event_type = msg->msg_type; + int pdes_msg_size = msg->value; + + //printf("==DIR[%d] DIR Registering dir_event_type:%d (time: %lf)\n", + // s->director_id, dir_registered_event_type, tw_now(lp)); + + s->nw_event_size[dir_registered_event_type] = pdes_msg_size; + memcpy(s->nw_event_ptr[dir_registered_event_type], &msg->buffer, pdes_msg_size); + + //int pdes_event_type = msg->op_type; + //nw_message *buffer = &msg->buffer; + //nw_message *saved_msg = s->nw_event_ptr[dir_registered_event_type]; + //printf("==DIR s->director_id: %d | dir_registered_event_type: %d | pdes_event_type: %d (%d)\n", + // s->director_id, dir_registered_event_type, pdes_event_type, + // buffer->msg_type); +} + + + + +// initializes the director LP +void dir_test_init(director_state* s, tw_lp* lp) +{ + // initialize the LP's and load the data + memset(s, 0, sizeof(*s)); + s->simulation_mode = SIM_MODE_PDES; + s->director_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + + for(int i = 0; i < DIR_MAX_PREDICTION; i++){ + s->predictions[i] = (tw_stime) 1000000; + } + + for(int i = 0; i < NUM_DIR_TO_NW_EVENT; i++){ + s->nw_event_ptr[i] = (void*) calloc(1, g_tw_msg_sz); + } + + // get lp_id of the nw that matches this director + int num_nw_per_mgrp; + s->nw_lpid; + num_nw_per_mgrp = codes_mapping_get_lp_count ("MODELNET_GRP", 1, "nw-lp", NULL, 0); + codes_mapping_get_lp_id("MODELNET_GRP", "nw-lp", NULL, 1, s->director_id / num_nw_per_mgrp, s->director_id % num_nw_per_mgrp, &(s->nw_lpid)); + + // Get switching criteria from configuration + // if we're switch based on iteration - read iter start and end + // if switch based on virtual time - schedule sending switch event to CODES + // (stage 2) if switch based on accuracy - schedule polling for accuracy + // (stage 2) pass training data from CODES to surrogate + // (stage 3) using workload with network surrogates + + // Update global configs + if(s->director_id == 1) + { + director_config_global.surr_iter_start = 1; + director_config_global.surr_iter_end = 2; + } + //printf("\n==DIR s->director_id: %d | lp->gid: %llu | s->nw_lpid: %llu", s->director_id, LLU(lp->gid), LLU(s->nw_lpid)); + return; +} + +void dir_test_event_handler(director_state* s, tw_bf * bf, director_message * m, tw_lp * lp) +{ + + switch(m->msg_type) + { + case DIR_OP_NW: + if(s->simulation_mode == SIM_MODE_PDES) + { + tw_error(TW_LOC, "DIR sent for non-annotation operation during PDES mode."); + } else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) + { + //printf("==DIR[%d] Skipping NW Op type:%d (time: %lf)\n", s->director_id, m->value, tw_now(lp)); + + tw_stime delay_ts = 0.001; + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); + } + break; + + case DIR_AN_ITER_MARK: + //printf("==DIR[%d] Non-blocking call (time %lf)\n", s->director_id, tw_now(lp)); + //printf("==DIR[%d] s->predictions[%d]: %lf\n", s->director_id, m->value, s->predictions[m->value]); + //fprintf(iteration_log, "DIR %d (time %lf)\n", s->director_id, tw_now(lp)); + //printf("==DIR[%d] DIR_AN_ITER_MARK m->value: %d (time: %lf)\n", s->director_id, m->value, tw_now(lp)); + if(s->simulation_mode == SIM_MODE_PDES) + { + if(m->value == director_config_global.surr_iter_start) + { + //printf("==DIR[%d] Triggering switch to SURR (time: %lf)\n", s->director_id, tw_now(lp)); + + s->simulation_mode = SIM_MODE_ITERATION_SURROGATE; + tw_stime delay_ts = s->predictions[m->value]; + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__SWITCH_TO_SURR, delay_ts, lp); + return; + } + else + { + tw_stime delay_ts = 0.001; + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); + return; + } + } + else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) + { + if(m->value == director_config_global.surr_iter_end) + { + //printf("==DIR[%d] Triggering switch to PDES (time: %lf)\n", s->director_id, tw_now(lp)); + + s->simulation_mode = SIM_MODE_PDES; + tw_stime delay_ts = 0.001; + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__SWITCH_TO_PDES, delay_ts, lp); + return; + } + else // we need to predict when the next iteration will start + { + tw_stime delay_ts = s->predictions[m->value]; + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); + return; + } + } + else + { + tw_error(TW_LOC, "[DIR] Simulation mode unknown."); + } + + break; + + case DIR_REGISTERED_EVENT__SWITCH_TO_SURR: + case DIR_REGISTERED_EVENT__SWITCH_TO_PDES: + case DIR_REGISTERED_EVENT__MOVE_TO_NEXT: + director_register_events(s, m, lp); + break; + + default: + break; + } +} + +void dir_test_finalize(director_state* s, tw_lp* lp) +{ + //printf("\n==DIR: FINALIZED"); +} + +tw_lptype dir_lp = { + (init_f) dir_test_init, + (pre_run_f) NULL, + (event_f) dir_test_event_handler, + (revent_f) NULL, //dir_test_event_handler_rc, + (commit_f) NULL, //dir_test_event_handler_commit, + (final_f) dir_test_finalize, + (map_f) codes_mapping, + sizeof(director_state) +}; + + + + +/* ========================================================== + END OF Director Code (To be moved to separate files) + ========================================================== +*/ + + + + /* turning on track lp will generate a lot of output messages */ #define DBG_COMM 1 #define MN_LP_NM "modelnet_dragonfly_custom" @@ -190,6 +472,8 @@ enum MPI_NW_EVENTS CLI_OTHER_FINISH, //received when another workload has finished // Surrogate events SURR_SKIP_ITERATION, // skips one (several) iteration(s) of simulation + CODES_CMD_SWITCH_TO_SURR, // transition simulation from PDES -> Surrogate + CODES_CMD_SWITCH_TO_PDES, // transition simulation from Surrogate -> PDES }; /* type of synthetic traffic */ @@ -366,6 +650,11 @@ struct nw_state char output_buf[512]; char col_stats[64]; struct ross_model_sample ross_sample; + + /* For hybrid simulations with DIRECTOR and surrogate */ + int director_enabled; + tw_lpid director_lpid; + int simulation_mode; }; /* data for handling reverse computation. @@ -418,6 +707,54 @@ struct nw_message } rc; }; + +/* ========================================================== + START of model-net replay DIRECTOR interface + ========================================================== +*/ + + +void codes_register_director_events(nw_state* s, int dir_event_type, int nw_event_type, size_t event_value, tw_lp* lp) +{ + tw_event *e; + director_message *msg; + + nw_message registered_nw_msg; + registered_nw_msg.msg_type = nw_event_type; + registered_nw_msg.op_type = event_value; + registered_nw_msg.fwd.app_id = s->nw_id; + + tw_stime ts = 0.001; + + //printf("==DIR[%d] NW Registering dir_event_type:%d | nw_event_type:%d (time: %lf)\n", + // s->nw_id, dir_event_type, nw_event_type, tw_now(lp)); + + //printf("==DIR: ts: %lf\n", ts); + if (sizeof(nw_message) + sizeof(director_message) > g_tw_msg_sz){ + tw_error(TW_LOC, "Error: NW-director trying to transmit an event of size " + "%d but ROSS is configured for events of size %zd\n", + sizeof(nw_message) + sizeof(director_message), g_tw_msg_sz); + } + + e = tw_event_new(s->director_lpid, ts, lp); + msg = (director_message*)tw_event_data(e); + msg->msg_type = dir_event_type; + msg->op_type = nw_event_type; + msg->value = sizeof(nw_message); + + memcpy(&msg->buffer, ®istered_nw_msg, msg->value); + + tw_event_send(e); +} + +/* ========================================================== + END of model-net replay DIRECTOR interface + ========================================================== +*/ + + + + static void send_ack_back(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp, mpi_msgs_queue * mpi_op, int matched_req); static void send_ack_back_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp); @@ -2406,6 +2743,23 @@ void nw_test_init(nw_state* s, tw_lp* lp) // This had been -1 but if qos is not configured (single job no workload conf file) // then this will error out + /* START of DIRECTOR setup + */ + s->director_enabled = 0; + s->simulation_mode = SIM_MODE_PDES; + int num_dir_per_mgrp = codes_mapping_get_lp_count ("MODELNET_GRP", 1, "dir-lp", NULL, 0); + if(num_dir_per_mgrp > 0){ + s->director_enabled = 1; + codes_mapping_get_lp_id("MODELNET_GRP", "dir-lp", NULL, 1, s->nw_id / num_dir_per_mgrp, s->nw_id % num_dir_per_mgrp, &s->director_lpid); + //printf("\n==DIRNW s->nw_id: %d | lp->gid: %llu | s>director_lpid: %llu", s->nw_id, LLU(lp->gid), LLU(s->director_lpid)); + } + // register callbacks with director + codes_register_director_events(s, DIR_REGISTERED_EVENT__SWITCH_TO_SURR, CODES_CMD_SWITCH_TO_SURR, sizeof(nw_message), lp); + codes_register_director_events(s, DIR_REGISTERED_EVENT__SWITCH_TO_PDES, CODES_CMD_SWITCH_TO_PDES, sizeof(nw_message), lp); + codes_register_director_events(s, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, MPI_OP_GET_NEXT, sizeof(nw_message), lp); + /* + * END of DIRECTOR setup */ + char type_name[512]; if(!num_net_traces) @@ -2758,6 +3112,33 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) case SURR_SKIP_ITERATION: skip_iteration(s, lp, bf, m); + break; + + /* START of Sim. transition events sent by DIRECTOR + */ + case CODES_CMD_SWITCH_TO_SURR: + if(s->simulation_mode == SIM_MODE_PDES){ + //printf("==DIR[%d] NW switch to SURR (time: %lf)\n", s->nw_id, tw_now(lp)); + + s->simulation_mode = SIM_MODE_ITERATION_SURROGATE; + get_next_mpi_operation(s, bf, m, lp); + }else{ + + } + break; + + case CODES_CMD_SWITCH_TO_PDES: + if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE){ + //printf("==DIR[%d] NW switch to PDES (time: %lf)\n", s->nw_id, tw_now(lp)); + + s->simulation_mode = SIM_MODE_PDES; + get_next_mpi_operation(s, bf, m, lp); + }else{ + + } + break; + /* + * END of Sim. transition event sent by DIRECTOR */ } } @@ -2899,6 +3280,34 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l // printf("Client rank %llu completed workload, local rank %d .\n", s->nw_id, s->local_rank); return; + } + if(mpi_op->op_type == CODES_WK_MARK) // If annotation type + {// TODO: extend this section to include checks for all annotations + m->rc.saved_marker_time = tw_now(lp); + + // If we have reached the surrogate switch time, skip next iteration(s) + if (have_we_hit_surrogate_switch(mpi_op)) { + tw_event *e = tw_event_new(lp->gid, 2076575.16 * 91, lp); + nw_message* msg = (nw_message*) tw_event_data(e); + msg->msg_type = SURR_SKIP_ITERATION; + tw_event_send(e); + } + + if(s->director_enabled == 1) + { + //printf("===DIR: Value=%d\n", mpi_op->u.send.tag); + codes_issue_director_event(lp, s->director_lpid, DIR_AN_ITER_MARK, mpi_op->u.send.tag); + //codes_issue_next_event(lp); + + return; + } + } else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) // Else non-annotation type + { + if(s->director_enabled == 1) + { + codes_issue_director_event(lp, s->director_lpid, DIR_OP_NW, mpi_op->op_type); + return; + } } switch(mpi_op->op_type) { @@ -2989,21 +3398,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l } break; - case CODES_WK_MARK: - { - m->rc.saved_marker_time = tw_now(lp); - - // If we have reached the surrogate switch time, skip next iteration(s) - if (have_we_hit_surrogate_switch(mpi_op)) { - tw_event *e = tw_event_new(lp->gid, 2076575.16 * 91, lp); - nw_message* msg = (nw_message*) tw_event_data(e); - msg->msg_type = SURR_SKIP_ITERATION; - tw_event_send(e); - } else { - codes_issue_next_event(lp); - } - } - break; + default: @@ -3583,6 +3978,7 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv ) if (g_st_ev_trace || g_st_model_stats || g_st_use_analysis_lps) nw_lp_register_model(); + lp_type_register("dir-lp", &dir_lp); // DIRECTOR addition - register type net_ids = model_net_configure(&num_nets); // assert(num_nets == 1); net_id = *net_ids; From 5ae2e7c0e437780e477a3bf19f9cedf37578624b Mon Sep 17 00:00:00 2001 From: "Kevin A. Brown" Date: Fri, 16 Aug 2024 18:39:43 +0000 Subject: [PATCH 03/13] director-b: complete initial director LP prototype for mpi-replay --- codes/surrogate/director-client.h | 105 ++++ src/CMakeLists.txt | 15 +- src/network-workloads/model-net-mpi-replay.c | 317 ++---------- src/surrogate/director-client.C | 505 +++++++++++++++++++ 4 files changed, 664 insertions(+), 278 deletions(-) create mode 100644 codes/surrogate/director-client.h create mode 100644 src/surrogate/director-client.C diff --git a/codes/surrogate/director-client.h b/codes/surrogate/director-client.h new file mode 100644 index 00000000..1e663e85 --- /dev/null +++ b/codes/surrogate/director-client.h @@ -0,0 +1,105 @@ +#ifndef __DIRECTOR_CLIENT_H_DEFINED__ +#define __DIRECTOR_CLIENT_H_DEFINED__ + +#include +#include "codes/codes_mapping.h" + + + +#define NUM_DIR_TO_NW_EVENT 20 + + +enum SIMULATION_MODE +{ + SIM_MODE_PDES=1, + SIM_MODE_ITERATION_SURROGATE, +}; + + +typedef struct director_message director_message; +typedef struct director_annotation director_annotation; + +enum DIR_EVENTS +{ + DIR_AN_ITER_MARK=1, + DIR_OP_NW, + DIR_REGISTERED_EVENT__SWITCH_TO_SURR, + DIR_REGISTERED_EVENT__SWITCH_TO_PDES, + DIR_REGISTERED_EVENT__MOVE_TO_NEXT, +}; + +enum DIR_OPERATIONS //currently unused +{ + DIR_AN_WK_START=1, + DIR_AN_WK_ITERATION_END, + DIR_AN_WK_END, + DIR_OP_SEND, + DIR_OP_RECV, +}; + + +// director event message struct +struct director_message +{ + int msg_type; + int op_type; + int num_rngs; + int value; + //model_net_event_return event_rc; + //struct codes_workload_op * mpi_op; + + void *buffer; // this pointer MUST be at the end of the structure +}; + +// director annotation struct +struct director_annotation +{ + int an_type; + int an_value; +}; + + +#ifdef __cplusplus +extern "C" +{ +#endif + + +/** + * @brief Prepares a request to send to client with the specified command and arguments, + * receives a reply + + * @param cmd zmqml request command: 'query', 'launch', execute', send', 'nothing', 'exit' + * @param args the arguments for launch and execute + * @param bindata binary data from send + * @param surrdata containing the 'status' field and optionally 'et' and 'id'. + * 'status' is not present, returns a vector with "failed". + * Fromat is ":;:;..." + * + */ + +//extern char* dir_client_request(const char* cmd, +// const char* args, +// const char* data); + + +extern void director_lp_register_model(const char *); + + +/* +extern void director_parse_args(char *args, int **args_array, int *length); +static void director_issue_codes_event(director_state * s, tw_lpid nw_lpid, int dir_registered_event_type, tw_stime ts, tw_lp* lp); +extern void director_register_events(director_state * s, director_message * msg, tw_lp * lp); +extern void dir_test_init(director_state* s, tw_lp* lp); +extern void director_prepare_iteration_dataset(director_state* s, tw_stime * training_data, int training_cycle, int training_records); +extern void director_get_surrogate_prediction(director_state* s, tw_bf * bf, director_message * m, tw_lp * lp, tw_stime* delay_ts); +extern void dir_test_event_handler(director_state* s, tw_bf * bf, director_message * m, tw_lp * lp); +extern void dir_test_finalize(director_state* s, tw_lp* lp); +*/ + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d82c2584..cd38259e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -59,6 +59,7 @@ list(APPEND SRCS surrogate/switch.c surrogate/packet-latency-predictor/common.c surrogate/packet-latency-predictor/average.c + surrogate/director-client.C iokernellang/codesparser.h iokernellang/codesparser.c @@ -155,6 +156,12 @@ if(USE_ONLINE) endif() endif() +# ZMQML +add_library(zmqmlrequester SHARED IMPORTED ) +set_target_properties(zmqmlrequester PROPERTIES + IMPORTED_LOCATION "${ZMQML_BUILD_PATH}/libzmqmlrequester.so" + INTERFACE_INCLUDE_DIRECTORIES "${ZMQML_BUILD_PATH}") + #LINK ROSS # target_link_libraries(codes PUBLIC #{pkgcfg_lib_ROSS_ROSS}) # target_link_libraries(codes PUBLIC PkgConfig::ROSS) @@ -166,6 +173,7 @@ target_include_directories(codes PUBLIC ${PROJECT_SOURCE_DIR}/codes ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/src/modelconfig + $ ) target_link_libraries(codes PUBLIC ${LIBS_TO_LINK}) @@ -198,9 +206,14 @@ if(USE_DUMPI) list(APPEND CODES_TARGETS model-net-dumpi-traces-dump) endif() +# ZMQ +pkg_check_modules(PC_ZeroMQ QUIET zmq) +find_path(ZeroMQ_INCLUDE_DIR NAMES zmq.hpp PATHS ${PC_ZeroMQ_INCLUDE_DIRS}) +find_library(ZeroMQ_LIBRARY NAMES zmq PATHS ${PC_ZeroMQ_LIBRARY_DIRS}) + foreach(tar IN LISTS CODES_TARGETS) target_include_directories(${tar} PUBLIC ${CODES_INCLUDE_DIRS} ${ROSS_INCLUDE_DIRS}) - target_link_libraries(${tar} PUBLIC codes ${LIBS_TO_LINK}) + target_link_libraries(${tar} PUBLIC codes ${LIBS_TO_LINK} zmqmlrequester ${ZeroMQ_LIBRARY}) endforeach() diff --git a/src/network-workloads/model-net-mpi-replay.c b/src/network-workloads/model-net-mpi-replay.c index eb1dc685..b2acc964 100644 --- a/src/network-workloads/model-net-mpi-replay.c +++ b/src/network-workloads/model-net-mpi-replay.c @@ -20,284 +20,17 @@ #include "codes/congestion-controller-core.h" +#include "codes/surrogate/director-client.h" /* ========================================================== START OF Director Code (To be moved to separate files) ========================================================== */ -struct -{ - int surr_iter_start; - int surr_iter_end; -} director_config_global; //struct director_config_struct dir_config_global; -#define DIR_MAX_PREDICTION 10 -#define NUM_DIR_TO_NW_EVENT 20 - -enum SIMULATION_MODE -{ - SIM_MODE_PDES=1, - SIM_MODE_ITERATION_SURROGATE, -}; - -typedef struct director_state director_state; -typedef struct director_message director_message; -typedef struct director_annotation director_annotation; - -enum DIR_EVENTS -{ - DIR_AN_ITER_MARK=1, - DIR_OP_NW, - DIR_REGISTERED_EVENT__SWITCH_TO_SURR, - DIR_REGISTERED_EVENT__SWITCH_TO_PDES, - DIR_REGISTERED_EVENT__MOVE_TO_NEXT, -}; - -enum DIR_OPERATIONS //currently unused -{ - DIR_AN_WK_START=1, - DIR_AN_WK_ITERATION_END, - DIR_AN_WK_END, - DIR_OP_SEND, - DIR_OP_RECV, -}; - -// state of the director LP -struct director_state -{ - tw_lpid director_id; - int simulation_mode; - - tw_stime predictions[DIR_MAX_PREDICTION]; - - void *nw_event_ptr[NUM_DIR_TO_NW_EVENT]; - int nw_event_size[NUM_DIR_TO_NW_EVENT]; - - tw_lpid nw_lpid; -}; - -// director event message struct -struct director_message -{ - int msg_type; - int op_type; - int num_rngs; - int value; - model_net_event_return event_rc; - struct codes_workload_op * mpi_op; - - void *buffer; // this pointer MUST be at the end of the structure -}; - -// director annotation struct -struct director_annotation -{ - int an_type; - int an_value; -}; - -/* Trigger Director Event from within network model*/ -static void codes_issue_director_event(tw_lp* lp, tw_lpid director_lpid, int dir_event_type, int value) -{ - - tw_event *e; - struct director_message* msg; - - tw_stime ts; - - ts = .0001; // Todo: maybe this should be dynamically adjustable - - e = tw_event_new(director_lpid, ts, lp ); - msg = (director_message*)tw_event_data(e); - - msg->msg_type = dir_event_type; - msg->value = value; - tw_event_send(e); -} - -/* Trigger CODES Event From Director */ -static void director_issue_codes_event(director_state * s, tw_lpid nw_lpid, int dir_registered_event_type, tw_stime ts, tw_lp* lp) -{ - - tw_event *e; - void* msg; - - //printf("==DIR: ts: %lf\n", ts); - e = tw_event_new(nw_lpid, ts, lp); - msg = (void*)tw_event_data(e); - - memcpy(msg, s->nw_event_ptr[dir_registered_event_type], s->nw_event_size[dir_registered_event_type]); - - //msg->msg_type = dir_registered_event_type; - tw_event_send(e); -} - -void director_register_events(director_state * s, director_message * msg, tw_lp * lp) -{ - int dir_registered_event_type = msg->msg_type; - int pdes_msg_size = msg->value; - - //printf("==DIR[%d] DIR Registering dir_event_type:%d (time: %lf)\n", - // s->director_id, dir_registered_event_type, tw_now(lp)); - - s->nw_event_size[dir_registered_event_type] = pdes_msg_size; - memcpy(s->nw_event_ptr[dir_registered_event_type], &msg->buffer, pdes_msg_size); - - //int pdes_event_type = msg->op_type; - //nw_message *buffer = &msg->buffer; - //nw_message *saved_msg = s->nw_event_ptr[dir_registered_event_type]; - //printf("==DIR s->director_id: %d | dir_registered_event_type: %d | pdes_event_type: %d (%d)\n", - // s->director_id, dir_registered_event_type, pdes_event_type, - // buffer->msg_type); -} - - - - -// initializes the director LP -void dir_test_init(director_state* s, tw_lp* lp) -{ - // initialize the LP's and load the data - memset(s, 0, sizeof(*s)); - s->simulation_mode = SIM_MODE_PDES; - s->director_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); - - for(int i = 0; i < DIR_MAX_PREDICTION; i++){ - s->predictions[i] = (tw_stime) 1000000; - } - - for(int i = 0; i < NUM_DIR_TO_NW_EVENT; i++){ - s->nw_event_ptr[i] = (void*) calloc(1, g_tw_msg_sz); - } - - // get lp_id of the nw that matches this director - int num_nw_per_mgrp; - s->nw_lpid; - num_nw_per_mgrp = codes_mapping_get_lp_count ("MODELNET_GRP", 1, "nw-lp", NULL, 0); - codes_mapping_get_lp_id("MODELNET_GRP", "nw-lp", NULL, 1, s->director_id / num_nw_per_mgrp, s->director_id % num_nw_per_mgrp, &(s->nw_lpid)); - - // Get switching criteria from configuration - // if we're switch based on iteration - read iter start and end - // if switch based on virtual time - schedule sending switch event to CODES - // (stage 2) if switch based on accuracy - schedule polling for accuracy - // (stage 2) pass training data from CODES to surrogate - // (stage 3) using workload with network surrogates - - // Update global configs - if(s->director_id == 1) - { - director_config_global.surr_iter_start = 1; - director_config_global.surr_iter_end = 2; - } - //printf("\n==DIR s->director_id: %d | lp->gid: %llu | s->nw_lpid: %llu", s->director_id, LLU(lp->gid), LLU(s->nw_lpid)); - return; -} - -void dir_test_event_handler(director_state* s, tw_bf * bf, director_message * m, tw_lp * lp) -{ - - switch(m->msg_type) - { - case DIR_OP_NW: - if(s->simulation_mode == SIM_MODE_PDES) - { - tw_error(TW_LOC, "DIR sent for non-annotation operation during PDES mode."); - } else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) - { - //printf("==DIR[%d] Skipping NW Op type:%d (time: %lf)\n", s->director_id, m->value, tw_now(lp)); - - tw_stime delay_ts = 0.001; - director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); - } - break; - - case DIR_AN_ITER_MARK: - //printf("==DIR[%d] Non-blocking call (time %lf)\n", s->director_id, tw_now(lp)); - //printf("==DIR[%d] s->predictions[%d]: %lf\n", s->director_id, m->value, s->predictions[m->value]); - //fprintf(iteration_log, "DIR %d (time %lf)\n", s->director_id, tw_now(lp)); - //printf("==DIR[%d] DIR_AN_ITER_MARK m->value: %d (time: %lf)\n", s->director_id, m->value, tw_now(lp)); - if(s->simulation_mode == SIM_MODE_PDES) - { - if(m->value == director_config_global.surr_iter_start) - { - //printf("==DIR[%d] Triggering switch to SURR (time: %lf)\n", s->director_id, tw_now(lp)); - - s->simulation_mode = SIM_MODE_ITERATION_SURROGATE; - tw_stime delay_ts = s->predictions[m->value]; - director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__SWITCH_TO_SURR, delay_ts, lp); - return; - } - else - { - tw_stime delay_ts = 0.001; - director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); - return; - } - } - else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) - { - if(m->value == director_config_global.surr_iter_end) - { - //printf("==DIR[%d] Triggering switch to PDES (time: %lf)\n", s->director_id, tw_now(lp)); - - s->simulation_mode = SIM_MODE_PDES; - tw_stime delay_ts = 0.001; - director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__SWITCH_TO_PDES, delay_ts, lp); - return; - } - else // we need to predict when the next iteration will start - { - tw_stime delay_ts = s->predictions[m->value]; - director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); - return; - } - } - else - { - tw_error(TW_LOC, "[DIR] Simulation mode unknown."); - } - - break; - - case DIR_REGISTERED_EVENT__SWITCH_TO_SURR: - case DIR_REGISTERED_EVENT__SWITCH_TO_PDES: - case DIR_REGISTERED_EVENT__MOVE_TO_NEXT: - director_register_events(s, m, lp); - break; - - default: - break; - } -} - -void dir_test_finalize(director_state* s, tw_lp* lp) -{ - //printf("\n==DIR: FINALIZED"); -} - -tw_lptype dir_lp = { - (init_f) dir_test_init, - (pre_run_f) NULL, - (event_f) dir_test_event_handler, - (revent_f) NULL, //dir_test_event_handler_rc, - (commit_f) NULL, //dir_test_event_handler_commit, - (final_f) dir_test_finalize, - (map_f) codes_mapping, - sizeof(director_state) -}; - - - - -/* ========================================================== - END OF Director Code (To be moved to separate files) - ========================================================== -*/ - @@ -723,6 +456,7 @@ void codes_register_director_events(nw_state* s, int dir_event_type, int nw_even registered_nw_msg.msg_type = nw_event_type; registered_nw_msg.op_type = event_value; registered_nw_msg.fwd.app_id = s->nw_id; + //registered_nw_msg.fwd.data_type = 9999; tw_stime ts = 0.001; @@ -747,6 +481,24 @@ void codes_register_director_events(nw_state* s, int dir_event_type, int nw_even tw_event_send(e); } +/* Trigger Director Event from within network model*/ +static void codes_issue_director_event(tw_lp* lp, tw_lpid director_lpid, int dir_event_type, int value) +{ + + tw_event *e; + struct director_message* msg; + + tw_stime ts; + + ts = .0001; // Todo: maybe this should be dynamically adjustable + + e = tw_event_new(director_lpid, ts, lp ); + msg = (director_message*)tw_event_data(e); + + msg->msg_type = dir_event_type; + msg->value = value; + tw_event_send(e); +} /* ========================================================== END of model-net replay DIRECTOR interface ========================================================== @@ -2747,17 +2499,22 @@ void nw_test_init(nw_state* s, tw_lp* lp) */ s->director_enabled = 0; s->simulation_mode = SIM_MODE_PDES; - int num_dir_per_mgrp = codes_mapping_get_lp_count ("MODELNET_GRP", 1, "dir-lp", NULL, 0); + int num_dir_per_mgrp = codes_mapping_get_lp_count ("MODELNET_GRP", 1, "dir-nw-lp", NULL, 0); if(num_dir_per_mgrp > 0){ s->director_enabled = 1; - codes_mapping_get_lp_id("MODELNET_GRP", "dir-lp", NULL, 1, s->nw_id / num_dir_per_mgrp, s->nw_id % num_dir_per_mgrp, &s->director_lpid); + codes_mapping_get_lp_id("MODELNET_GRP", "dir-nw-lp", NULL, 1, s->nw_id / num_dir_per_mgrp, s->nw_id % num_dir_per_mgrp, &s->director_lpid); + + // register callbacks with director + codes_register_director_events(s, DIR_REGISTERED_EVENT__SWITCH_TO_SURR, CODES_CMD_SWITCH_TO_SURR, sizeof(nw_message), lp); + codes_register_director_events(s, DIR_REGISTERED_EVENT__SWITCH_TO_PDES, CODES_CMD_SWITCH_TO_PDES, sizeof(nw_message), lp); + codes_register_director_events(s, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, MPI_OP_GET_NEXT, sizeof(nw_message), lp); //printf("\n==DIRNW s->nw_id: %d | lp->gid: %llu | s>director_lpid: %llu", s->nw_id, LLU(lp->gid), LLU(s->director_lpid)); + } - // register callbacks with director - codes_register_director_events(s, DIR_REGISTERED_EVENT__SWITCH_TO_SURR, CODES_CMD_SWITCH_TO_SURR, sizeof(nw_message), lp); - codes_register_director_events(s, DIR_REGISTERED_EVENT__SWITCH_TO_PDES, CODES_CMD_SWITCH_TO_PDES, sizeof(nw_message), lp); - codes_register_director_events(s, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, MPI_OP_GET_NEXT, sizeof(nw_message), lp); - /* + else{ + //printf("\n==NW[%d] DIRECTOR: No director LPs found (dir-nw-lp)\n", s->nw_id); + } + /* * END of DIRECTOR setup */ char type_name[512]; @@ -3082,6 +2839,8 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) break; case MPI_OP_GET_NEXT: + //if(m->fwd.data_type == 9999) + // printf("===[%llu] N-MARK[%llu]: Value=%d\n", lp->gid, 1, 9999); get_next_mpi_operation(s, bf, m, lp); break; @@ -3297,8 +3056,11 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l { //printf("===DIR: Value=%d\n", mpi_op->u.send.tag); codes_issue_director_event(lp, s->director_lpid, DIR_AN_ITER_MARK, mpi_op->u.send.tag); - //codes_issue_next_event(lp); + //printf("===[%llu] N-MARK[%llu]: Value=%d\n", lp->gid, s->director_lpid, mpi_op->u.send.tag); + return; + }else{ + codes_issue_next_event(lp); return; } } else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) // Else non-annotation type @@ -3978,7 +3740,8 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv ) if (g_st_ev_trace || g_st_model_stats || g_st_use_analysis_lps) nw_lp_register_model(); - lp_type_register("dir-lp", &dir_lp); // DIRECTOR addition - register type + director_lp_register_model("dir-nw-lp"); + net_ids = model_net_configure(&num_nets); // assert(num_nets == 1); net_id = *net_ids; diff --git a/src/surrogate/director-client.C b/src/surrogate/director-client.C new file mode 100644 index 00000000..c3738064 --- /dev/null +++ b/src/surrogate/director-client.C @@ -0,0 +1,505 @@ +#include +#include +#include +#include +#include + +#include +#include +#include // std::min_element +#include //std::fixed +#include // std::precision + +#include "codes/surrogate/director-client.h" +#include "codes/configuration.h" +#include "zmqmlrequester.h" + + +#define NUM_ACTIVE_CLIENTS 72 //TODO: this should be calculated at runtime + +#define DIR_ZMQ_CMD_LENGTH 15 +#define DIR_ZMQ_ARG_LENGTH 100 + +#define DIR_MAX_PREDICTION 5 +#define DIR_MAX_TRAINING_RECORDS 10 +#define DIR_MAX_DATA_SIZE 15 + +struct +{ + int surr_iter_start; + int surr_iter_end; +} director_config_global; + +typedef struct director_state director_state; + +// Some flag to relocate/clean-up +int evaluate_perf = 1; +int training_enabled = 0; //TODO: Move this to the LP state +int surrogate_enabled = 0; +int inferencing_enabled = 1; + + +std::vector total_elapsed_times; +std::vector zmq_processing_times; + +// state of the director LP +struct director_state +{ + tw_lpid director_id; + int simulation_mode; + + int training_cycle_id; + int training_record_id; + tw_stime training_data[DIR_MAX_TRAINING_RECORDS]; + //std::vector training_data_vc; + + int next_prediction_index; + tw_stime predictions[DIR_MAX_PREDICTION]; + + void *nw_event_ptr[NUM_DIR_TO_NW_EVENT]; + int nw_event_size[NUM_DIR_TO_NW_EVENT]; + + tw_lpid nw_lpid; +}; + +std::vector director_get_str_list(const char *s, const char delimiter) { + std::vector result; + std::stringstream ss (s); + std::string item; + + while (getline (ss, item, delimiter)) { result.push_back (item); } + return result; +} +std::string director_get_list_str(std::vector s, const char delimiter) { + std::ostringstream mergedstr; + std::copy(s.begin(), s.end(), std::ostream_iterator(mergedstr, " ")); + std::cout << mergedstr.str() << std::endl; + return mergedstr.str(); +} + +std::vector director_client_request( + const char* cmd, + const char* args, + const std::string data) +{ + std::vector ret; + /* + std::cout << cmd << " ARGS " << args << std::endl; + //if(strcmp(cmd, "send-records") == 0){ + std::cout << data << std::endl; + //} + */ + if(strcmp(cmd, "exit") == 0){ + ret = zmqml_request(cmd); + return ret; + } + + auto start_time = std::chrono::steady_clock::now(); // TODO - find a way to enclose this in evaluate_perf? + + std::vector args_list; + args_list = director_get_str_list(args, ';'); + ret = zmqml_request(cmd, args_list, data); + + if (evaluate_perf == 1){ + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration(end_time - start_time).count(); + + total_elapsed_times.push_back(duration); + zmq_processing_times.push_back( std::stod(ret[1]) ); + + if(zmq_processing_times.size() == NUM_ACTIVE_CLIENTS){ + double sum = 0; + for (double ts : zmq_processing_times) sum += ts; + double mean = sum / zmq_processing_times.size(); + double sum_sq_diff = 0; + for (double ts : zmq_processing_times) sum_sq_diff += (ts - mean) * (ts - mean); + double std_dev = sqrt(sum_sq_diff / zmq_processing_times.size()); + auto min = std::min_element(zmq_processing_times.begin(), zmq_processing_times.end()); + auto max = std::max_element(zmq_processing_times.begin(), zmq_processing_times.end()); + + zmq_processing_times.clear(); + + std::cout << std::setprecision(9) << std::fixed; + /* + std::cout << "ZMQ_VALS: "; + for(auto d: zmq_processing_times) + std::cout << d << " ;"; + std::cout << std::endl; + */ + std::cout << "==DIR_STATS zmq-processing: " << cmd + << " latency: mean = " << mean + << ", min = " << *min + << ", max = " << *max + << ", std-deviation = " << std_dev + << std::endl; + + double tsum = 0; + for (double ts : total_elapsed_times) tsum += ts; + double tmean = tsum / total_elapsed_times.size(); + double tsum_sq_diff = 0; + for (double ts : total_elapsed_times) tsum_sq_diff += (ts - tmean) * (ts - tmean); + double tstd_dev = sqrt(tsum_sq_diff / total_elapsed_times.size()); + auto tmin = std::min_element(total_elapsed_times.begin(), total_elapsed_times.end()); + auto tmax = std::max_element(total_elapsed_times.begin(), total_elapsed_times.end()); + /* + std::cout << "TOTAL_VALS: "; + for(auto d: total_elapsed_times) + std::cout << d << " ;"; + std::cout << std::endl; + */ + total_elapsed_times.clear(); + + std::cout << "==DIR_STATS zmq-total: " << cmd + << " latency: mean = " << tmean + << ", min = " << *tmin + << ", max = " << *tmax + << ", std-deviation = " << tstd_dev + << std::endl; + } + } + /* + std::cout << cmd << "|" << args << " | "; + for(auto s: ret) + std::cout << s << " ;"; + std::cout << std::endl; + */ + + return ret; +} + + +/* Trigger CODES Event From Director */ +static void director_issue_codes_event(director_state * s, tw_lpid nw_lpid, int dir_registered_event_type, tw_stime ts, tw_lp* lp) +{ + + tw_event *e; + void* msg; + + //printf("==DIR: ts: %lf\n", ts); + e = tw_event_new(nw_lpid, ts, lp); + msg = (void*)tw_event_data(e); + + memcpy(msg, s->nw_event_ptr[dir_registered_event_type], s->nw_event_size[dir_registered_event_type]); + + //msg->msg_type = dir_registered_event_type; + tw_event_send(e); +} + +void director_register_events(director_state * s, director_message * msg, tw_lp * lp) +{ + int dir_registered_event_type = msg->msg_type; + int pdes_msg_size = msg->value; + + //printf("==DIR[%d] DIR Registering dir_event_type:%d (time: %lf)\n", + // s->director_id, dir_registered_event_type, tw_now(lp)); + + s->nw_event_size[dir_registered_event_type] = pdes_msg_size; + memcpy(s->nw_event_ptr[dir_registered_event_type], &msg->buffer, pdes_msg_size); + + //int pdes_event_type = msg->op_type; + //nw_message *buffer = &msg->buffer; + //nw_message *saved_msg = s->nw_event_ptr[dir_registered_event_type]; + //printf("==DIR s->director_id: %d | dir_registered_event_type: %d | pdes_event_type: %d (%d)\n", + // s->director_id, dir_registered_event_type, pdes_event_type, + // buffer->msg_type); +} + + + + +// initializes the director LP +void director_init(director_state* s, tw_lp* lp) +{ + // initialize the LP's and load the data + memset(s, 0, sizeof(*s)); + s->simulation_mode = SIM_MODE_PDES; + s->director_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + + s->training_cycle_id = 0; + s->training_record_id = 1; + s->training_data[0] = tw_now(lp); + //s->training_data_vc.push_back(tw_now(lp)); + s->next_prediction_index = -1; + for(int i = 0; i < DIR_MAX_PREDICTION; i++){ + s->predictions[i] = (tw_stime) 1000000; + } + + for(int i = 0; i < NUM_DIR_TO_NW_EVENT; i++){ + s->nw_event_ptr[i] = (void*) calloc(1, g_tw_msg_sz); + } + + // get lp_id of the nw that matches this director + int num_nw_per_mgrp; + s->nw_lpid; + num_nw_per_mgrp = codes_mapping_get_lp_count ("MODELNET_GRP", 1, "nw-lp", NULL, 0); + codes_mapping_get_lp_id("MODELNET_GRP", "nw-lp", NULL, 1, s->director_id / num_nw_per_mgrp, s->director_id % num_nw_per_mgrp, &(s->nw_lpid)); + + // Get switching criteria from configuration + // if we're switch based on iteration - read iter start and end + // if switch based on virtual time - schedule sending switch event to CODES + // (stage 2) if switch based on accuracy - schedule polling for accuracy + // (stage 2) pass training data from CODES to surrogate + // (stage 3) using workload with network surrogates + + // Update global configs + if(s->director_id == 1) + { + int rc = 1, rc1 = 1, rc2 = 1; + rc = configuration_get_value_int(&config, "DIRECTOR", "surrogate_enabled", NULL, &surrogate_enabled); + if(rc) + surrogate_enabled = 0; + if(surrogate_enabled){ + rc1 = configuration_get_value_int(&config, "DIRECTOR", "start_iter", NULL, &director_config_global.surr_iter_start); + rc2 = configuration_get_value_int(&config, "DIRECTOR", "end_iter", NULL, &director_config_global.surr_iter_end); + if(rc1 || rc2){ + director_config_global.surr_iter_start = 100000; + director_config_global.surr_iter_end = 100001; + surrogate_enabled = 0; + } + } + + rc = configuration_get_value_int(&config, "DIRECTOR", "inferencing_enabled", NULL, &inferencing_enabled); + if(rc) + inferencing_enabled = 0; + + rc = configuration_get_value_int(&config, "DIRECTOR", "training_enabled", NULL, &training_enabled); + if(rc) + training_enabled = 0; + } + //printf("\n==DIR s->director_id: %d | lp->gid: %llu | s->nw_lpid: %llu", s->director_id, LLU(lp->gid), LLU(s->nw_lpid)); + + /* + char commandstr[DIR_ZMQ_CMD_LENGTH]; + char args[DIR_ZMQ_ARG_LENGTH]; + std::vector ret_vals; + + sprintf(commandstr,"cmd-%d", s->director_id); + sprintf(args, "2;args-1;"); + + ret_vals = director_client_request(commandstr, args, ""); + //printf("UNIVERSE: %s - %s\n", commandstr, ret_vals[0]); + */ + + return; +} + +void director_prepare_iteration_dataset(director_state* s, tw_stime * training_data, int training_cycle, int training_records) +{ + //printf("==DIR[%d] Training Cycle: %d\n", s->director_id, training_cycle); + + std::string processed_training_data_str; + int i, length = 0; + double tmp_data = 0.0; + char tmp_data_str[DIR_MAX_DATA_SIZE]; + int written = 0; + + // Prepare dataset + for(i = 1; i < training_records; i++) + { + tmp_data = training_data[i] - training_data[i-1]; + written += sprintf(tmp_data_str, "%.2f;", tmp_data); + //strcat(processed_training_data_str, tmp_data_str); + + processed_training_data_str.append(std::to_string(tmp_data)); + processed_training_data_str.append(" "); + // printf(" %3d: %lf [%lf]\n", i, training_data[i], tmp_data); + } + //std::cout << "Processed Data: " << processed_training_data_str << std::endl; + + // Send dataset + std::vector ret_vals; + char commandstr[DIR_ZMQ_CMD_LENGTH]; + char args[DIR_ZMQ_ARG_LENGTH]; + + sprintf(commandstr, "send-records"); + sprintf(args, "%d;%d;%d", 3, s->director_id, training_records - 1); // num-of-args;num-record + ret_vals = director_client_request(commandstr, args, processed_training_data_str); +} + +void director_get_surrogate_prediction(director_state* s, tw_bf * bf, director_message * m, tw_lp * lp, tw_stime* delay_ts) +{ + // Check if we have sufficient predictions + if(s->next_prediction_index == -1){ // we need more + //printf("==DIR[%d] DIR Prediction -- generating set (time: %lf)\n", + // s->director_id, tw_now(lp)); + + if(inferencing_enabled){ + // Pull more predictions + std::vector ret_vals; + char commandstr[15]; + char args[100]; + + sprintf(commandstr, "do-inference"); + sprintf(args, "%d;%d;%d;", 3, s->director_id, DIR_MAX_PREDICTION); // num-of-args;num-record + std::string input_data = ("1000000 1000000 1000000 1000000"); + + ret_vals = director_client_request(commandstr, args, input_data); + /* + std::cout << "PREDICTIONS: " << commandstr + << " [0]" << ret_vals[0] + << " [1]" << ret_vals[1] + << " [2]" << ret_vals[2] + << std::endl; + */ + std::vector predictions = director_get_str_list(ret_vals[2].c_str(), ' '); + + //std::cout << "PREDICTIONS: " << predictions.size() << " | "; + int i = 0; + for(auto p: predictions){ + //std::cout << " " << std::stof(p); + s->predictions[i] = std::stof(p); + i += 1; + } + //std::cout << std::endl; + assert(i <= DIR_MAX_PREDICTION); + } + + s->next_prediction_index = 0; + } + *delay_ts = s->predictions[s->next_prediction_index]; + + s->next_prediction_index = s->next_prediction_index + 1; + + // Check if we've exhuasted the predictions + if(s->next_prediction_index == DIR_MAX_PREDICTION){ + s->next_prediction_index = -1; + } +} + + +void director_event_handler(director_state* s, tw_bf * bf, director_message * m, tw_lp * lp) +{ + + switch(m->msg_type) + { + case DIR_OP_NW: + if(s->simulation_mode == SIM_MODE_PDES) + { + tw_error(TW_LOC, "DIR sent for non-annotation operation during PDES mode."); + } else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) + { + //printf("==DIR[%d] Skipping NW Op type:%d (time: %lf)\n", s->director_id, m->value, tw_now(lp)); + + tw_stime delay_ts = 0.001; + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); + } + break; + + case DIR_AN_ITER_MARK: + //fprintf(iteration_log, "DIR %d (time %lf)\n", s->director_id, tw_now(lp)); + //printf("==DIR[%d] DIR_AN_ITER_MARK m->value: %d (time: %lf)\n", s->director_id, m->value, tw_now(lp)); + if(s->simulation_mode == SIM_MODE_PDES) + { + // Manage training data + if(training_enabled && s->training_record_id < DIR_MAX_TRAINING_RECORDS) + {// There is space to store more training data + s->training_data[s->training_record_id] = tw_now(lp); + //s->training_data_vc.push_back(tw_now(lp)); + s->training_record_id = s->training_record_id + 1; + } + if(training_enabled && s->training_record_id == DIR_MAX_TRAINING_RECORDS) + {// We've filled all training data slots + //printf("==DIR[%d] Sending training dataset (time: %lf)\n", s->director_id, tw_now(lp)); + + // Prepare and send training data + director_prepare_iteration_dataset(s, s->training_data, s->training_cycle_id, DIR_MAX_TRAINING_RECORDS); + + // Increment cycle counter, reset record counter, and prime dataset + s->training_cycle_id = s->training_cycle_id + 1; + s->training_record_id = 1; + s->training_data[0] = tw_now(lp); + } + if(surrogate_enabled && m->value == director_config_global.surr_iter_start) + { + //printf("==DIR[%d] Triggering switch to SURR (time: %lf)\n", s->director_id, tw_now(lp)); + + s->simulation_mode = SIM_MODE_ITERATION_SURROGATE; + tw_stime delay_ts; + director_get_surrogate_prediction(s, bf, m, lp, &delay_ts); + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__SWITCH_TO_SURR, delay_ts, lp); + return; + } + else + { + tw_stime delay_ts = 0.001; + //printf("===[%llu] D-MARK[%llu]: Value=%d\n", s->nw_lpid, lp->gid, m->value ); + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); + return; + } + } + else if(s->simulation_mode == SIM_MODE_ITERATION_SURROGATE) + { + if(m->value == director_config_global.surr_iter_end) + { + //printf("==DIR[%d] Triggering switch to PDES (time: %lf)\n", s->director_id, tw_now(lp)); + + s->simulation_mode = SIM_MODE_PDES; + tw_stime delay_ts = 0.001; + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__SWITCH_TO_PDES, delay_ts, lp); + + if(training_enabled){ + // Restart training data collection + //s->training_data_vc.clear(); + s->training_data[0] = tw_now(lp); + s->training_record_id = 1; + } + return; + } + else // we need to predict when the next iteration will start + { + tw_stime delay_ts; + director_get_surrogate_prediction(s, bf, m, lp, &delay_ts); + director_issue_codes_event(s, s->nw_lpid, DIR_REGISTERED_EVENT__MOVE_TO_NEXT, delay_ts, lp); + return; + } + } + else + { + tw_error(TW_LOC, "[DIR] Simulation mode unknown."); + } + + break; + + case DIR_REGISTERED_EVENT__SWITCH_TO_SURR: + case DIR_REGISTERED_EVENT__SWITCH_TO_PDES: + case DIR_REGISTERED_EVENT__MOVE_TO_NEXT: + director_register_events(s, m, lp); + break; + + default: + break; + } +} + +void director_finalize(director_state* s, tw_lp* lp) +{ + if (s->director_id == 0 && (training_enabled || inferencing_enabled)) + director_client_request("exit", "", ""); + + //printf("\n==DIR: FINALIZED"); +} + +tw_lptype dir_lp = { + (init_f) director_init, + (pre_run_f) NULL, + (event_f) director_event_handler, + (revent_f) NULL, //director_event_handler_rc, + (commit_f) NULL, //director_event_handler_commit, + (final_f) director_finalize, + (map_f) codes_mapping, + sizeof(director_state) +}; + +extern void director_lp_register_model(const char * dir_lp_name){ + int num_dir_per_mgrp = codes_mapping_get_lp_count ("MODELNET_GRP", 1, "dir-nw-lp", NULL, 0); + if(num_dir_per_mgrp > 0){ + lp_type_register(dir_lp_name, &dir_lp); // DIRECTOR addition - register type + //printf("\n==DIR: Registered\n"); + } +} + + +/* ========================================================== + END OF Director Code (To be moved to separate files) + ========================================================== +*/ From 138f46a7d8ebd70ab3157ed8ea348f1f4f5cf91b Mon Sep 17 00:00:00 2001 From: "Kevin A. Brown" Date: Fri, 16 Aug 2024 18:58:43 +0000 Subject: [PATCH 04/13] zmqml: update zmq server and requester to interface with director LP --- src/surrogate/zmqml/zmqmlrequester.cpp | 5 ++ src/surrogate/zmqml/zmqmlserver.py | 68 +++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/surrogate/zmqml/zmqmlrequester.cpp b/src/surrogate/zmqml/zmqmlrequester.cpp index 6f43758c..a8ebbd53 100644 --- a/src/surrogate/zmqml/zmqmlrequester.cpp +++ b/src/surrogate/zmqml/zmqmlrequester.cpp @@ -20,6 +20,7 @@ using namespace rapidjson; static string endpoint = "tcp://localhost:5555"; static int debug = 0; + /** * See zmqmlrequester.h */ @@ -82,6 +83,10 @@ vector zmqml_request(const string& cmd, if (response.HasMember("id")) { ret.push_back(response["id"].GetString()); } + + if (response.HasMember("predictions")) { + ret.push_back(response["predictions"].GetString()); + } } else { ret.push_back("failed"); } diff --git a/src/surrogate/zmqml/zmqmlserver.py b/src/surrogate/zmqml/zmqmlserver.py index 066b0512..4e8105ea 100755 --- a/src/surrogate/zmqml/zmqmlserver.py +++ b/src/surrogate/zmqml/zmqmlserver.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # zmqmlserver : ZeroMQ-based ML task dispatching server @@ -13,6 +13,7 @@ import time from itertools import count # generate unit id # from dataclasses import dataclass +import numpy as np # TODO: abstract a mechanism to call training from runmlpacketdelay import run_mlpacketdelay_training @@ -24,13 +25,15 @@ endpoint = "tcp://*:5555" debug = False - +debug2 = False # # # launch_id = count(start=1) # unique for launched thread launched_threads = {} # id:obj. keep track of active threads. remove the thread once it finished +training_records = {} # client_id:[] + class LaunchCMD: def __init__(self): # thread event @@ -153,6 +156,61 @@ def receivedata(args, bindata): elapsed_time = time.time() - st return (status, elapsed_time) +# +# receive training records +# +def receiverecords(args, bindata): + status = "failed" + st = time.time() + + num_args = int(args[0]) # 1st arg is num of args + client = int(args[1]) # 2nd arg is client id + num_records = int(args[2]) # 3rd arg is num records + records_str = str(bindata.decode('utf-8')) + records_str = records_str.strip() + records = list(records_str.split(" ")) + + if client not in training_records: + training_records[client] = [] + + training_records[client].extend([float(s) for s in records]) + + if (debug2) and (client == 51): + print(f"Training records[51] :{training_records[client]}") + + status = "done" + elapsed_time = time.time() - st + return (status, elapsed_time) + + +# +# do inference to get predictions +# +def launch_surrogate_inferencing(args, bindata): + status = "failed" + st = time.time() + + num_args = int(args[0]) # 1st arg is num of args + client = int(args[1]) # 2nd arg is client id + num_steps = int(args[2]) # 3rd arg is num steps to predict + records_str = str(bindata.decode('utf-8')) + records_str = records_str.strip() + records = list(records_str.split(" ")) + + input_records = [float(s) for s in records] + + inferences = [] + for i in range(num_steps): + inferences.append(2000000.0) + + inferences_str = ' '.join([str(f) for f in inferences]) + + status = "done" + elapsed_time = time.time() - st + return (status, elapsed_time, inferences_str) + + +# # # main listener loop @@ -192,6 +250,12 @@ def zmq_cmd_listener(): destfn = args[0] (status, et) = receivedata(args, bindata) retmsg = {"status":status, "et":str(et)} + elif cmd == "send-records": + (status, et) = receiverecords(args, bindata) + retmsg = {"status":status, "et":str(et)} + elif cmd == "do-inference": + (status, et, predictions) = launch_surrogate_inferencing(args, bindata) + retmsg = {"status":status, "et":str(et), "predictions": predictions} # send response back to the requester socket.send_json(retmsg) From 39fbc4fb405087ba199a6264f051e056ab980b04 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 21 May 2026 10:33:23 -0400 Subject: [PATCH 05/13] Fix zmq and ROSS compilation issues The kronos-develop-director-b branch of CODES was using an outdated version of ROSS and also had compilation issues because of zeromq. This commit changes it to be compatible with the master branch of ROSS and fixes the zeromq compilation issues. --- CODES-compile-instructions.sh | 173 ++++++++++++++++++++++++++++++++++ codes/surrogate/switch.h | 2 +- src/CMakeLists.txt | 2 +- src/surrogate/init.c | 6 +- src/surrogate/switch.c | 61 ++++++------ src/surrogate/zmqml/Makefile | 2 +- src/util/rc-stack.c | 2 +- 7 files changed, 213 insertions(+), 35 deletions(-) create mode 100644 CODES-compile-instructions.sh diff --git a/CODES-compile-instructions.sh b/CODES-compile-instructions.sh new file mode 100644 index 00000000..c477e738 --- /dev/null +++ b/CODES-compile-instructions.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Switches +swm_enable=0 +union_enable=0 +torch_enable=1 + +# Uncomment below for MPICH +#export PATH=/usr/local/mpich-4.1.2/bin/:"$PATH" +# Note: remember to compile MPICH with nemesis not with UCX support + +################## Actual scripts starts from here ################## + +# SWM has to be enabled for UNION to work +if [ $union_enable = 1 ]; then + swm_enable=1 +fi + +# What to compile +CUR_DIR="$PWD" + +##### Downloading everything ##### + +if [ ! -d codes/.git ]; then + git clone https://github.com/codes-org/codes --depth=100 --branch=v1.5.0 +else + echo "Using existing codes checkout: $(realpath codes)" +fi + +if [ ! -d ross/.git ]; then + git clone https://github.com/ross-org/ross --depth=100 --branch=v8.1.0 +else + echo "Using existing ross checkout: $(realpath ross)" +fi + +if [ $swm_enable = 1 ]; then + git clone https://github.com/pmodels/argobots --depth=1 + git clone https://github.com/codes-org/swm-workloads --branch=v1.2 +fi + +if [ $union_enable = 1 ]; then + # Downloading conceptual + curl -L https://sourceforge.net/projects/conceptual/files/conceptual/1.5.1b/conceptual-1.5.1b.tar.gz -o conceptual-1.5.1b.tar.gz + tar xvf conceptual-1.5.1b.tar.gz + # Downloading union + git clone https://github.com/SPEAR-UIC/Union + pushd Union && git checkout 99b3df3 && popd +fi + +##### COMPILING ##### + +mkdir -p ross/build +pushd ross/build +cmake .. -DROSS_BUILD_MODELS=ON -DCMAKE_INSTALL_PREFIX="$(realpath ./bin)" \ + -DCMAKE_C_COMPILER=mpicc -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-g -Wall" +#make VERBOSE=1 +make install -j4 +err=$? +[[ $err -ne 0 ]] && exit $err +popd + +if [ $swm_enable = 1 ]; then + pushd swm-workloads/swm + ./prepare.sh + mkdir -p build + pushd build + ../configure --disable-shared --prefix="$(realpath ./bin)" CC=mpicc CXX=mpicxx CFLAGS=-g CXXFLAGS=-g + #make V=1 && make install + make -j4 && make install + err=$? + [[ $err -ne 0 ]] && exit $err + popd && popd + + pushd argobots + ./autogen.sh + mkdir -p build + pushd build + #../configure --enable-debug=all --disable-fast --disable-shared --prefix="$(realpath ./bin)" CC=mpicc CXX=mpicxx CFLAGS=-g CXXFLAGS=-g + ../configure --disable-shared --prefix="$(realpath ./bin)" CC=mpicc CXX=mpicxx CFLAGS=-g CXXFLAGS=-g + #make V=1 && make install + make -j4 && make install + err=$? + [[ $err -ne 0 ]] && exit $err + popd && popd +fi + +if [ $union_enable = 1 ]; then + pushd conceptual-1.5.1b + PYTHON=python2 ./configure --prefix="$(realpath ./install)" LIBS=-lm + make -j4 && make install + err=$? + [[ $err -ne 0 ]] && exit $err + popd + + pushd Union + # Python 2 override. Union expects Python 2 ONLY + mkdir -p python-override + ln -s /usr/bin/python2 python-override/python + # compiling + ./prepare.sh + PYTHON=python2 ./configure --disable-shared --with-conceptual="$(realpath ../conceptual-1.5.1b/install)" --with-conceptual-src="$(realpath ../conceptual-1.5.1b)" --prefix="$(realpath ./install)" CC=mpicc CXX=mpicxx + PATH="$PWD/python-override:$PATH" make -j4 && make install + err=$? + [[ $err -ne 0 ]] && exit $err + popd +fi + + +# Build local ZMQML requester library required by director-client.C +pushd codes/src/surrogate/zmqml +make clean +make +test -f libzmqmlrequester.so +test -f zmqmlrequester.h +popd + +# Make imported zmqmlrequester target visible to doc/example and tests. +python3 - <<'INNERPY' +from pathlib import Path +cm = Path("codes/src/CMakeLists.txt") +text = cm.read_text() +old = "add_library(zmqmlrequester SHARED IMPORTED )" +new = "add_library(zmqmlrequester SHARED IMPORTED GLOBAL)" +if old in text: + cm.write_text(text.replace(old, new)) +elif new in text: + pass +else: + raise SystemExit("Could not find zmqmlrequester imported target line in codes/src/CMakeLists.txt") +INNERPY + +mkdir -p codes/build +pushd codes/build + +make_args_codes=( + -DCMAKE_PREFIX_PATH="$(realpath "$CUR_DIR/ross/build/bin")" + -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc + -DCMAKE_C_FLAGS="-g -Wall" + -DCMAKE_CXX_FLAGS="-g -Wall" + -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON + -DCMAKE_INSTALL_PREFIX="$(realpath bin)" + -DZMQML_BUILD_PATH="$(realpath "$CUR_DIR/codes/src/surrogate/zmqml")" + -DZeroMQ_INCLUDE_DIR=/usr/include + -DZeroMQ_LIBRARY=/usr/lib/x86_64-linux-gnu/libzmq.so +) +if [ $swm_enable = 1 ]; then + make_args_codes=( + "${make_args_codes[@]}" + -DSWM_PKG_CONFIG_PATH="$(realpath "$CUR_DIR/swm-workloads/swm/build/maint")" + -DARGOBOTS_PKG_CONFIG_PATH="$(realpath "$CUR_DIR/argobots/build/maint")" + ) +fi +if [ $union_enable = 1 ]; then + make_args_codes=( + "${make_args_codes[@]}" + -DUNION_PKG_CONFIG_PATH="$(realpath "$CUR_DIR/Union/install/lib/pkgconfig")" + ) +fi +if [ $torch_enable = 1 ]; then + make_args_codes=("${make_args_codes[@]}" -DUSE_TORCH=true) +else + make_args_codes=("${make_args_codes[@]}" -DUSE_TORCH=false) +fi + +cmake .. "${make_args_codes[@]}" +#make VERBOSE=1 +make -j4 +err=$? +[[ $err -ne 0 ]] && exit $err + +popd diff --git a/codes/surrogate/switch.h b/codes/surrogate/switch.h index 553f3a11..82a31cf4 100644 --- a/codes/surrogate/switch.h +++ b/codes/surrogate/switch.h @@ -61,7 +61,7 @@ extern struct switch_at_struct switch_at; // Switch -void director_switch(tw_pe * pe, tw_event_sig gvt_sig); +void director_switch(tw_pe * pe, bool past_end_time); #ifdef __cplusplus } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cd38259e..9439ce2f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -157,7 +157,7 @@ if(USE_ONLINE) endif() # ZMQML -add_library(zmqmlrequester SHARED IMPORTED ) +add_library(zmqmlrequester SHARED IMPORTED GLOBAL) set_target_properties(zmqmlrequester PROPERTIES IMPORTED_LOCATION "${ZMQML_BUILD_PATH}/libzmqmlrequester.so" INTERFACE_INCLUDE_DIRECTORIES "${ZMQML_BUILD_PATH}") diff --git a/src/surrogate/init.c b/src/surrogate/init.c index 79bb7e71..42049d72 100644 --- a/src/surrogate/init.c +++ b/src/surrogate/init.c @@ -64,14 +64,14 @@ void surrogate_configure( PRINTF_ONCE("\n"); // Injecting into ROSS the function to be called at GVT and the instant in time to trigger GVT - g_tw_gvt_arbitrary_fun = director_switch; + g_tw_gvt_hook = director_switch; #ifdef USE_RAND_TIEBREAKER tw_event_sig time_stamp = {0}; time_stamp.recv_ts = switch_at.time_stampts[0]; - tw_trigger_arbitrary_fun_at(time_stamp); + tw_trigger_gvt_hook_at_event_sig(time_stamp); #else - tw_trigger_arbitrary_fun_at(switch_at.time_stampts[0]); + tw_trigger_gvt_hook_at(switch_at.time_stampts[0]); #endif // freeing timestamps before it dissapears diff --git a/src/surrogate/switch.c b/src/surrogate/switch.c index 4b29ab18..a906e152 100644 --- a/src/surrogate/switch.c +++ b/src/surrogate/switch.c @@ -79,9 +79,9 @@ static void rollback_and_cancel_events_pe(tw_pe * pe, tw_event_sig gvt_sig) { tw_stime const gvt = gvt_sig.recv_ts; // Backtracking the simulation to GVT for (unsigned int i = 0; i < g_tw_nkp; i++) { - tw_kp_rollback_to_sig(g_tw_kp[i], gvt_sig); + tw_kp_rollback_to_sig(g_tw_kp[i], &gvt_sig); } - assert(tw_event_sig_compare(pe->GVT_sig, gvt_sig) == 0); + assert(tw_event_sig_compare_ptr(&pe->GVT_sig, &gvt_sig) == 0); assert(pe->GVT_sig.recv_ts == gvt); // redundant but needed because compiler cries that gvt is never used #else static void rollback_and_cancel_events_pe(tw_pe * pe, tw_stime gvt) { @@ -100,10 +100,7 @@ static void rollback_and_cancel_events_pe(tw_pe * pe, tw_stime gvt) { pe->stats.s_net_read += tw_clock_read() - start; } - pe->gvt_status = 1; - tw_sched_event_q(pe); - tw_sched_cancel_q(pe); - tw_gvt_step2(pe); + tw_scheduler_rollback_and_cancel_events_pe(pe); if (DEBUG_DIRECTOR > 1) { printf("PE %lu: Time stamp at the end of GVT time: %f - AVL-tree sized: %d\n", g_tw_mynode, gvt, pe->avl_tree_size); @@ -146,7 +143,7 @@ static void shift_events_to_future_pe(tw_pe * pe, tw_stime gvt) { // Filtering events to freeze assert(next_event->prev == NULL); #ifdef USE_RAND_TIEBREAKER - assert(tw_event_sig_compare(next_event->sig, gvt_sig) >= 0); + assert(tw_event_sig_compare_ptr(&next_event->sig, &gvt_sig) >= 0); #else assert(next_event->recv_ts >= gvt); #endif @@ -165,11 +162,11 @@ static void shift_events_to_future_pe(tw_pe * pe, tw_stime gvt) { next_event->recv_ts += switch_offset; next_event->sig.recv_ts = next_event->recv_ts; } - assert(next_event->recv_ts >= g_tw_trigger_arbitrary_fun.sig_at.recv_ts); + assert(next_event->recv_ts >= g_tw_gvt_hook_trigger.sig_at.recv_ts); #else next_event->recv_ts += switch_offset; } - assert(next_event->recv_ts >= g_tw_trigger_arbitrary_fun.at); + assert(next_event->recv_ts >= g_tw_gvt_hook_trigger.at); #endif // store event in deque_events to inject immediately back to the queue @@ -382,11 +379,12 @@ static void events_surrogate_to_high_def_switch(tw_pe * pe, tw_stime gvt) { } +void director_switch(tw_pe * pe, bool past_end_time) { #ifdef USE_RAND_TIEBREAKER -void director_switch(tw_pe * pe, tw_event_sig gvt_sig) { + tw_event_sig const gvt_sig = pe->GVT_sig; tw_stime const gvt = gvt_sig.recv_ts; #else -void director_switch(tw_pe * pe, tw_stime gvt) { + tw_stime const gvt = pe->GVT; #endif assert(is_surrogate_configured); @@ -400,15 +398,18 @@ void director_switch(tw_pe * pe, tw_stime gvt) { printf("GVT %d at %f in %s arbitrary-fun-status=", i++, gvt, surr_config.director.is_surrogate_on() ? "surrogate-mode" : "high-definition"); - switch (g_tw_trigger_arbitrary_fun.active) { - case ARBITRARY_FUN_enabled: - printf("enabled\n"); + switch (g_tw_gvt_hook_trigger.status) { + case GVT_HOOK_STATUS_timestamp: + printf("timestamp\n"); break; - case ARBITRARY_FUN_disabled: + case GVT_HOOK_STATUS_disabled: printf("disabled\n"); break; - case ARBITRARY_FUN_triggered: - printf("triggered\n"); + case GVT_HOOK_STATUS_every_n_gvt: + printf("every-n-gvt\n"); + break; + case GVT_HOOK_STATUS_model_call: + printf("model-call\n"); break; } } @@ -430,16 +431,20 @@ void director_switch(tw_pe * pe, tw_stime gvt) { return; } - // Detecting if we are going to switch - if (switch_at.current_i < switch_at.total - && g_tw_trigger_arbitrary_fun.active == ARBITRARY_FUN_triggered) { + // Detecting if we are going to switch. + // + // Newer ROSS calls g_tw_gvt_hook only after the timestamp trigger fires, + // and it sets g_tw_gvt_hook_trigger.status back to GVT_HOOK_STATUS_disabled + // before entering this hook. Therefore, do not check for the old + // ARBITRARY_FUN_triggered state here; it no longer exists. + if (switch_at.current_i < switch_at.total) { double const switch_time = switch_at.time_stampts[switch_at.current_i]; #ifdef USE_RAND_TIEBREAKER - assert(g_tw_trigger_arbitrary_fun.sig_at.recv_ts == switch_at.time_stampts[switch_at.current_i]); + assert(g_tw_gvt_hook_trigger.sig_at.recv_ts == switch_time); #else - assert(g_tw_trigger_arbitrary_fun.at == switch_at.time_stampts[switch_at.current_i]); + assert(g_tw_gvt_hook_trigger.at == switch_time); #endif - assert(gvt >= switch_time); // current gvt shouldn't be that far ahead from the point we wanted to trigger it + assert(gvt >= switch_time); // current gvt should not be before the requested switch time } else { return; } @@ -457,10 +462,10 @@ void director_switch(tw_pe * pe, tw_stime gvt) { // Rollback if in optimistic mode #ifdef USE_RAND_TIEBREAKER if (g_tw_synchronization_protocol == OPTIMISTIC) { - assert(tw_event_sig_compare(pe->GVT_sig, gvt_sig) == 0); + assert(tw_event_sig_compare_ptr(&pe->GVT_sig, &gvt_sig) == 0); rollback_and_cancel_events_pe(pe, gvt_sig); - //assert(tw_event_sig_compare(pe->GVT_sig, gvt_sig) <= 0); - assert(tw_event_sig_compare(pe->GVT_sig, gvt_sig) == 0); + //assert(tw_event_sig_compare_ptr(&pe->GVT_sig, &gvt_sig) <= 0); + assert(tw_event_sig_compare_ptr(&pe->GVT_sig, &gvt_sig) == 0); } #else if (g_tw_synchronization_protocol == OPTIMISTIC) { @@ -502,10 +507,10 @@ void director_switch(tw_pe * pe, tw_stime gvt) { tw_event_sig time_stamp = {0}; time_stamp.recv_ts = next_switch; //printf("Adding a trigger to activate next switch!\n"); - tw_trigger_arbitrary_fun_at(time_stamp); + tw_trigger_gvt_hook_at_event_sig(time_stamp); #else //printf("Adding a trigger to activate next switch!\n"); - tw_trigger_arbitrary_fun_at(next_switch); + tw_trigger_gvt_hook_at(next_switch); #endif } diff --git a/src/surrogate/zmqml/Makefile b/src/surrogate/zmqml/Makefile index 4c28ed54..b4abcfab 100644 --- a/src/surrogate/zmqml/Makefile +++ b/src/surrogate/zmqml/Makefile @@ -7,7 +7,7 @@ TARGETS=libzmqmlrequester.so demozmqmlrequester all: $(TARGETS) libzmqmlrequester.so: zmqmlrequester.o - $(CXX) -shared -o $@ $^ + $(CXX) -shared -o $@ $^ $(LDFLAGS) zmqmlrequester.o: zmqmlrequester.cpp zmqmlrequester.h $(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ diff --git a/src/util/rc-stack.c b/src/util/rc-stack.c index ebb2131f..5f68123e 100644 --- a/src/util/rc-stack.c +++ b/src/util/rc-stack.c @@ -107,7 +107,7 @@ void rc_stack_gc(tw_lp const *lp, struct rc_stack *s) { while (ent != &s->head) { rc_entry *r = qlist_entry(ent, rc_entry, ql); #ifdef USE_RAND_TIEBREAKER - if (lp == NULL || tw_event_sig_compare(r->e_sig, lp->pe->GVT_sig) == -1) { + if (lp == NULL || tw_event_sig_compare_ptr(&r->e_sig, &lp->pe->GVT_sig) == -1) { #else if (lp == NULL || r->time < lp->pe->GVT){ #endif From 0651b5ec34855b6ed258f046363682ff3ec1befa Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 21 May 2026 13:58:01 -0400 Subject: [PATCH 06/13] Fix torch-jit compilation Compilation with torch-jit was not occuring even with torch_enable set to 1. This commit fixes torch-jit compilation with GPU support. --- CODES-compile-instructions.sh | 115 +++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/CODES-compile-instructions.sh b/CODES-compile-instructions.sh index c477e738..46f27b8c 100644 --- a/CODES-compile-instructions.sh +++ b/CODES-compile-instructions.sh @@ -134,8 +134,99 @@ INNERPY mkdir -p codes/build pushd codes/build +torch_cmake_prefix="" +torch_dir="" + +if [ "$torch_enable" = 1 ]; then + torch_cmake_prefix="$(python3 - <<'INNERPY' +import torch +print(torch.utils.cmake_prefix_path) +INNERPY +)" + torch_dir="${torch_cmake_prefix}/Torch" + + if [ ! -f "${torch_dir}/TorchConfig.cmake" ]; then + echo "ERROR: TorchConfig.cmake not found at: ${torch_dir}/TorchConfig.cmake" >&2 + echo " torch.utils.cmake_prefix_path returned: ${torch_cmake_prefix}" >&2 + exit 1 + fi + + echo "Using Torch CMake prefix: ${torch_cmake_prefix}" + echo "Using Torch_DIR: ${torch_dir}" + + # Optional CUDA toolkit override for CUDA-enabled PyTorch. + # Set CUDA_HOME before running this script, e.g.: + # export CUDA_HOME=/usr/local/cuda-12.4 + # or: + # export CUDA_HOME=/usr/local/cuda + if python3 - <<'INNERPY' +import torch, sys +sys.exit(0 if torch.version.cuda is not None else 1) +INNERPY + then + if [ -z "${CUDA_HOME:-}" ]; then + if [ -d /usr/local/cuda ]; then + CUDA_HOME=/usr/local/cuda + else + echo "ERROR: CUDA-enabled PyTorch detected, but CUDA_HOME is not set and /usr/local/cuda does not exist." >&2 + echo " Set CUDA_HOME to your CUDA toolkit root, e.g. /usr/local/cuda-12.4." >&2 + exit 1 + fi + fi + + if [ ! -f "${CUDA_HOME}/include/cuda_runtime_api.h" ]; then + echo "ERROR: Missing CUDA header: ${CUDA_HOME}/include/cuda_runtime_api.h" >&2 + exit 1 + fi + + if [ ! -f "${CUDA_HOME}/lib64/libcudart.so" ] && [ ! -f "${CUDA_HOME}/lib/libcudart.so" ]; then + echo "ERROR: Missing CUDA runtime library under ${CUDA_HOME}/lib64 or ${CUDA_HOME}/lib" >&2 + exit 1 + fi + + if [ ! -x "${CUDA_HOME}/bin/nvcc" ]; then + echo "ERROR: Missing CUDA compiler: ${CUDA_HOME}/bin/nvcc" >&2 + exit 1 + fi + + if [ ! -d "${CUDA_HOME}/nvvm/libdevice" ]; then + echo "ERROR: Missing CUDA libdevice directory: ${CUDA_HOME}/nvvm/libdevice" >&2 + exit 1 + fi + + cuda_arch="" + if command -v nvidia-smi >/dev/null 2>&1; then + cuda_arch="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -n1 | tr -d '.[:space:]' || true)" + fi + + if [ -z "${cuda_arch}" ]; then + echo "WARNING: Could not auto-detect GPU compute capability with nvidia-smi." >&2 + echo " Falling back to CMAKE_CUDA_ARCHITECTURES=80." >&2 + cuda_arch="80" + fi + + export CUDA_HOME + export CUDA_PATH="${CUDA_HOME}" + export CUDA_ROOT="${CUDA_HOME}" + export CUDA_TOOLKIT_ROOT_DIR="${CUDA_HOME}" + export CUDAToolkit_ROOT="${CUDA_HOME}" + export CUDACXX="${CUDA_HOME}/bin/nvcc" + export PATH="${CUDA_HOME}/bin:${PATH}" + export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${CUDA_HOME}/lib:${LD_LIBRARY_PATH:-}" + + echo "Using CUDA_HOME: ${CUDA_HOME}" + echo "Using CUDACXX: ${CUDACXX}" + echo "Using CMAKE_CUDA_ARCHITECTURES=${cuda_arch}" + fi +fi + +cmake_prefix_path="$(realpath "$CUR_DIR/ross/build/bin")" +if [ "$torch_enable" = 1 ]; then + cmake_prefix_path="${cmake_prefix_path};${torch_cmake_prefix}" +fi + make_args_codes=( - -DCMAKE_PREFIX_PATH="$(realpath "$CUR_DIR/ross/build/bin")" + -DCMAKE_PREFIX_PATH="${cmake_prefix_path}" -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_C_FLAGS="-g -Wall" -DCMAKE_CXX_FLAGS="-g -Wall" @@ -158,8 +249,26 @@ if [ $union_enable = 1 ]; then -DUNION_PKG_CONFIG_PATH="$(realpath "$CUR_DIR/Union/install/lib/pkgconfig")" ) fi -if [ $torch_enable = 1 ]; then - make_args_codes=("${make_args_codes[@]}" -DUSE_TORCH=true) +if [ "$torch_enable" = 1 ]; then + make_args_codes=( + "${make_args_codes[@]}" + -DUSE_TORCH=true + -DTorch_DIR="${torch_dir}" + ) + + if [ -n "${CUDA_HOME:-}" ]; then + make_args_codes=( + "${make_args_codes[@]}" + -DCUDA_TOOLKIT_ROOT_DIR="${CUDA_HOME}" + -DCUDAToolkit_ROOT="${CUDA_HOME}" + -DCUDA_PATH="${CUDA_HOME}" + -DCUDA_ROOT="${CUDA_HOME}" + -DCMAKE_CUDA_COMPILER="${CUDA_HOME}/bin/nvcc" + -DCMAKE_CUDA_ARCHITECTURES="${cuda_arch}" + -DCUDA_INCLUDE_DIRS="${CUDA_HOME}/include" + -DCUDA_CUDART_LIBRARY="${CUDA_HOME}/lib64/libcudart.so" + ) + fi else make_args_codes=("${make_args_codes[@]}" -DUSE_TORCH=false) fi From 01a2b16c3bf4c38990262808ecc9bee8c9be4477 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 21 May 2026 14:50:42 -0400 Subject: [PATCH 07/13] Allow cpu-based PyTorch usage --- CODES-compile-instructions.sh | 72 ++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/CODES-compile-instructions.sh b/CODES-compile-instructions.sh index 46f27b8c..1ba6f3c6 100644 --- a/CODES-compile-instructions.sh +++ b/CODES-compile-instructions.sh @@ -154,47 +154,56 @@ INNERPY echo "Using Torch CMake prefix: ${torch_cmake_prefix}" echo "Using Torch_DIR: ${torch_dir}" - # Optional CUDA toolkit override for CUDA-enabled PyTorch. - # Set CUDA_HOME before running this script, e.g.: + # CUDA is intentionally opt-in. + # Default to CPU-only Torch-JIT compilation unless CUDA_HOME is explicitly set. + # + # To enable CUDA, run for example: # export CUDA_HOME=/usr/local/cuda-12.4 - # or: - # export CUDA_HOME=/usr/local/cuda - if python3 - <<'INNERPY' -import torch, sys -sys.exit(0 if torch.version.cuda is not None else 1) + # ./CODES-compile-instructions.sh + torch_cuda_version="$(python3 - <<'INNERPY' +import torch +print(torch.version.cuda or "") INNERPY - then - if [ -z "${CUDA_HOME:-}" ]; then - if [ -d /usr/local/cuda ]; then - CUDA_HOME=/usr/local/cuda - else - echo "ERROR: CUDA-enabled PyTorch detected, but CUDA_HOME is not set and /usr/local/cuda does not exist." >&2 - echo " Set CUDA_HOME to your CUDA toolkit root, e.g. /usr/local/cuda-12.4." >&2 - exit 1 - fi - fi +)" + + cuda_arch="" + if [ -z "${CUDA_HOME:-}" ] && [ -n "${torch_cuda_version}" ]; then + echo "ERROR: CUDA_HOME is not set, so this script is defaulting to CPU-only Torch-JIT compilation." >&2 + echo " However, the active Python environment has a CUDA-enabled PyTorch build:" >&2 + echo " torch.version.cuda=${torch_cuda_version}" >&2 + echo "" >&2 + echo " CMake cannot use a CUDA-enabled PyTorch package as a CPU-only LibTorch package." >&2 + echo " Choose one of the following:" >&2 + echo " 1. For CPU-only compilation, install a CPU-only PyTorch build in this environment." >&2 + echo " 2. For CUDA compilation, export CUDA_HOME to your CUDA toolkit root." >&2 + echo "" >&2 + echo " Example CUDA build:" >&2 + echo " export CUDA_HOME=/usr/local/cuda-12.4" >&2 + echo " bash CODES-compile-instructions.sh" >&2 + exit 1 + fi + if [ -n "${CUDA_HOME:-}" ]; then if [ ! -f "${CUDA_HOME}/include/cuda_runtime_api.h" ]; then - echo "ERROR: Missing CUDA header: ${CUDA_HOME}/include/cuda_runtime_api.h" >&2 + echo "ERROR: CUDA_HOME is set, but missing CUDA header: ${CUDA_HOME}/include/cuda_runtime_api.h" >&2 exit 1 fi if [ ! -f "${CUDA_HOME}/lib64/libcudart.so" ] && [ ! -f "${CUDA_HOME}/lib/libcudart.so" ]; then - echo "ERROR: Missing CUDA runtime library under ${CUDA_HOME}/lib64 or ${CUDA_HOME}/lib" >&2 + echo "ERROR: CUDA_HOME is set, but missing CUDA runtime library under ${CUDA_HOME}/lib64 or ${CUDA_HOME}/lib" >&2 exit 1 fi if [ ! -x "${CUDA_HOME}/bin/nvcc" ]; then - echo "ERROR: Missing CUDA compiler: ${CUDA_HOME}/bin/nvcc" >&2 + echo "ERROR: CUDA_HOME is set, but missing CUDA compiler: ${CUDA_HOME}/bin/nvcc" >&2 exit 1 fi if [ ! -d "${CUDA_HOME}/nvvm/libdevice" ]; then - echo "ERROR: Missing CUDA libdevice directory: ${CUDA_HOME}/nvvm/libdevice" >&2 + echo "ERROR: CUDA_HOME is set, but missing CUDA libdevice directory: ${CUDA_HOME}/nvvm/libdevice" >&2 exit 1 fi - cuda_arch="" if command -v nvidia-smi >/dev/null 2>&1; then cuda_arch="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -n1 | tr -d '.[:space:]' || true)" fi @@ -214,9 +223,22 @@ INNERPY export PATH="${CUDA_HOME}/bin:${PATH}" export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${CUDA_HOME}/lib:${LD_LIBRARY_PATH:-}" + echo "CUDA_HOME is set; enabling CUDA Torch-JIT compilation." echo "Using CUDA_HOME: ${CUDA_HOME}" echo "Using CUDACXX: ${CUDACXX}" echo "Using CMAKE_CUDA_ARCHITECTURES=${cuda_arch}" + else + echo "CUDA_HOME is not set; forcing CPU-only Torch-JIT compilation." + + # Prevent accidental CUDA discovery from /usr/local/cuda, nvcc on PATH, + # inherited CMake cache variables, or CUDA-enabled PyTorch metadata. + unset CUDA_HOME + unset CUDA_PATH + unset CUDA_ROOT + unset CUDA_TOOLKIT_ROOT_DIR + unset CUDAToolkit_ROOT + unset CUDACXX + unset CMAKE_CUDA_COMPILER fi fi @@ -268,6 +290,12 @@ if [ "$torch_enable" = 1 ]; then -DCUDA_INCLUDE_DIRS="${CUDA_HOME}/include" -DCUDA_CUDART_LIBRARY="${CUDA_HOME}/lib64/libcudart.so" ) + else + make_args_codes=( + "${make_args_codes[@]}" + -DCMAKE_DISABLE_FIND_PACKAGE_CUDA=ON + -DCMAKE_DISABLE_FIND_PACKAGE_CUDAToolkit=ON + ) fi else make_args_codes=("${make_args_codes[@]}" -DUSE_TORCH=false) From e42e75a89c80882558f7b2f40c8d6e8900833ace Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 27 May 2026 10:48:25 -0400 Subject: [PATCH 08/13] Move ML models to surrogate directory --- .../ml_models}/combine_packet_latency_traces_across_ranks.py | 0 .../surrogate/ml_models}/train_packet_latency_torchjit.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {ml_models => src/surrogate/ml_models}/combine_packet_latency_traces_across_ranks.py (100%) rename {ml_models => src/surrogate/ml_models}/train_packet_latency_torchjit.py (100%) diff --git a/ml_models/combine_packet_latency_traces_across_ranks.py b/src/surrogate/ml_models/combine_packet_latency_traces_across_ranks.py similarity index 100% rename from ml_models/combine_packet_latency_traces_across_ranks.py rename to src/surrogate/ml_models/combine_packet_latency_traces_across_ranks.py diff --git a/ml_models/train_packet_latency_torchjit.py b/src/surrogate/ml_models/train_packet_latency_torchjit.py similarity index 100% rename from ml_models/train_packet_latency_torchjit.py rename to src/surrogate/ml_models/train_packet_latency_torchjit.py From 4a50395a31420ba8f577a378c346ba156dde3d67 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 27 May 2026 10:21:02 -0400 Subject: [PATCH 09/13] surrogate: support LP-aware Torch-JIT latency model Expand the Torch-JIT packet latency predictor to use an LP-aware feature vector with terminal, packet, LP gid, router/group, queue, VC occupancy, and processing-delay context. Update pure PDES trace export so the generated training CSV records the true ROSS caller LP gid, keeping training data consistent with runtime inference. Update the component-level training and runtime feature construction paths to use the expanded feature order for the two model targets: travel_end_time_delta and next_packet_delay. --- .../packet-latency-predictor/common.h | 12 + .../packet-latency-predictor/torch-jit.h | 2 + .../tutorial-ping-pong-surrogate.conf.in | 2 +- src/networks/model-net/dragonfly-dally.C | 188 ++++++++- src/surrogate/init.c | 9 +- ...bine_packet_latency_traces_across_ranks.py | 77 ++++ .../train_lp_aware_packet_latency_model.py | 371 ++++++++++++++++++ ...bine_packet_latency_traces_across_ranks.py | 0 .../director/lp_aware_packet_latency_model.py | 106 +++++ .../train_packet_latency_torchjit.py | 0 .../packet-latency-predictor/average.c | 16 + .../packet-latency-predictor/torch-jit.C | 112 +++++- 12 files changed, 868 insertions(+), 27 deletions(-) create mode 100644 src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py create mode 100755 src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py rename src/surrogate/ml_models/{ => director}/combine_packet_latency_traces_across_ranks.py (100%) create mode 100644 src/surrogate/ml_models/director/lp_aware_packet_latency_model.py rename src/surrogate/ml_models/{ => director}/train_packet_latency_torchjit.py (100%) diff --git a/codes/surrogate/packet-latency-predictor/common.h b/codes/surrogate/packet-latency-predictor/common.h index 3faa7bff..a84e20c1 100644 --- a/codes/surrogate/packet-latency-predictor/common.h +++ b/codes/surrogate/packet-latency-predictor/common.h @@ -28,6 +28,18 @@ struct packet_start { double processing_packet_delay; // delay for this packet to be processed from previous packet in the queue uint32_t packet_size; bool is_there_another_pckt_in_queue; // is there another packet in queue + uint64_t caller_lp_gid; // true ROSS LP gid of the source terminal LP + + /* + * Optional ML-facing context for LP-aware Torch-JIT mode. + * Existing predictors may ignore these fields. + */ + uint32_t src_router_id; + uint32_t src_group_id; + uint32_t dst_router_id; + uint32_t dst_group_id; + uint32_t terminal_queue_length; + uint32_t terminal_vc_occupancy; }; struct packet_end { diff --git a/codes/surrogate/packet-latency-predictor/torch-jit.h b/codes/surrogate/packet-latency-predictor/torch-jit.h index 80e532a5..cb1031b7 100644 --- a/codes/surrogate/packet-latency-predictor/torch-jit.h +++ b/codes/surrogate/packet-latency-predictor/torch-jit.h @@ -2,12 +2,14 @@ #define CODES_SURROGATE_TORCHJIT_H #include +#include #include "codes/surrogate/init.h" #ifdef __cplusplus extern "C" { #endif +void surrogate_torch_set_lp_aware_mode(bool enabled); void surrogate_torch_init(char const * dir); extern struct packet_latency_predictor torch_latency_predictor; diff --git a/doc/example/tutorial-ping-pong-surrogate.conf.in b/doc/example/tutorial-ping-pong-surrogate.conf.in index fd53f4d1..bdf9be33 100644 --- a/doc/example/tutorial-ping-pong-surrogate.conf.in +++ b/doc/example/tutorial-ping-pong-surrogate.conf.in @@ -76,7 +76,7 @@ NETWORK_SURROGATE { ignore_until="${IGNORE_UNTIL}"; # parameters for torch-jit latency predictor - torch_jit_mode="single-static-model-for-all-terminals"; + torch_jit_mode="${TORCH_JIT_MODE}"; torch_jit_model_path="${TORCH_JIT_MODEL_PATH}"; # selecting network treatment on switching to surrogate. Options: frezee, nothing diff --git a/src/networks/model-net/dragonfly-dally.C b/src/networks/model-net/dragonfly-dally.C index 91befa1b..eba3ab1b 100644 --- a/src/networks/model-net/dragonfly-dally.C +++ b/src/networks/model-net/dragonfly-dally.C @@ -102,6 +102,8 @@ static long global_stalled_chunk_counter = 0; #define OUTPUT_SNAPSHOT 1 static int num_snapshots = 0; + + tw_stime * snapshot_times; char snapshot_filename[128]; @@ -2485,6 +2487,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params) if (rc_enable > 0) { enable_network_surrogate = (strcmp(enable_str, "1") == 0 || strcmp(enable_str, "true") == 0); } + // if surrogate mode has been set up if (enable_network_surrogate) { struct network_surrogate_config surr_conf = { @@ -2571,7 +2574,12 @@ static void setup_packet_latency_path(char const * const dir_to_save) { tw_error(TW_LOC, "File %s could not be opened", filename_path); } - fprintf(packet_latency_f, "#src_terminal,dest_terminal,packet_id,is_surrogate_on,is_predicted,size,workload_injection,next_packet_delay,start,end,latency,is_there_another_pckt_in_queue\n"); + fprintf(packet_latency_f, + "#src_terminal,dest_terminal,packet_size,is_there_another_pckt_in_queue," + "caller_lp_gid,src_router_id,src_group_id,dst_router_id,dst_group_id," + "terminal_queue_length,terminal_vc_occupancy,processing_packet_delay," + "travel_end_time_delta,next_packet_delay," + "packet_id,is_surrogate_on,is_predicted,workload_injection,start,end,latency\n"); } /* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */ @@ -3019,6 +3027,94 @@ static int get_next_router_vcg(router_state * s, tw_bf * bf, terminal_dally_mess return -1; } +static uint32_t dfdally_clamp_u64_to_u32(uint64_t value) +{ + uint64_t const max_u32 = (uint64_t)((uint32_t)-1); + return value > max_u32 ? (uint32_t)-1 : (uint32_t)value; +} + +static unsigned int dfdally_safe_rail_id( + terminal_state const *s, + terminal_dally_message const *msg) +{ + if (!s || !s->params || s->params->num_rails <= 0) { + return 0; + } + + if (msg && msg->rail_id >= 0 && msg->rail_id < s->params->num_rails) { + return (unsigned int)msg->rail_id; + } + + return 0; +} + +static uint32_t dfdally_terminal_queue_length_bytes(terminal_state const *s) +{ + if (!s || !s->params || !s->terminal_length) { + return (uint32_t)-1; + } + + uint64_t total = 0; + for (int rail = 0; rail < s->params->num_rails; rail++) { + if (!s->terminal_length[rail]) { + continue; + } + for (int qos = 0; qos < s->params->num_qos_levels; qos++) { + if (s->terminal_length[rail][qos] > 0) { + total += (uint64_t)s->terminal_length[rail][qos]; + } + } + } + + return dfdally_clamp_u64_to_u32(total); +} + +static uint32_t dfdally_terminal_vc_occupancy_bytes(terminal_state const *s) +{ + if (!s || !s->params || !s->vc_occupancy) { + return (uint32_t)-1; + } + + uint64_t total = 0; + for (int rail = 0; rail < s->params->num_rails; rail++) { + if (!s->vc_occupancy[rail]) { + continue; + } + for (int qos = 0; qos < s->params->num_qos_levels; qos++) { + if (s->vc_occupancy[rail][qos] > 0) { + total += (uint64_t)s->vc_occupancy[rail][qos]; + } + } + } + + return dfdally_clamp_u64_to_u32(total); +} + +static uint32_t dfdally_terminal_src_router_id( + terminal_state const *s, + terminal_dally_message const *msg) +{ + if (!s || !s->params) { + return (uint32_t)-1; + } + + unsigned int const rail_id = dfdally_safe_rail_id(s, msg); + + /* + * In high-fidelity mode, s->router_id is valid. In frozen surrogate mode, + * dragonfly_dally_terminal_highdef_to_surrogate() zeroes most network + * state, so s->router_id may be NULL. Average mode still goes through + * packet_generate_predicted(), so do not dereference s->router_id unless + * it exists. + */ + if (s->router_id) { + return s->router_id[rail_id]; + } + + return dfdally_get_assigned_router_id_from_terminal( + s->params, s->terminal_id, rail_id); +} + static inline void packet_latency_save_to_file( unsigned int terminal_id, struct packet_start * start, @@ -3028,13 +3124,30 @@ static inline void packet_latency_save_to_file( ) { if (!packet_latency_f) { return; } // Don't save if there isn't a file to save to if (end->travel_end_time > g_tw_ts_end) { return; } // This packet could never arrive to its destination! - fprintf(packet_latency_f, "%u,%u,%lu,%d,%d,%u,%f,%f,%f,%f,%f,%d\n", - terminal_id, start->dfdally_dest_terminal_id, start->packet_ID, - surrogate_on, is_predicted, start->packet_size, + double const latency = end->travel_end_time - start->travel_start_time; + fprintf(packet_latency_f, + "%u,%u,%u,%d,%llu,%u,%u,%u,%u,%u,%u,%f,%f,%f,%lu,%d,%d,%f,%f,%f,%f\n", + terminal_id, + start->dfdally_dest_terminal_id, + start->packet_size, + start->is_there_another_pckt_in_queue, + (unsigned long long)start->caller_lp_gid, + start->src_router_id, + start->src_group_id, + start->dst_router_id, + start->dst_group_id, + start->terminal_queue_length, + start->terminal_vc_occupancy, + start->processing_packet_delay, + latency, + end->next_packet_delay, + start->packet_ID, + surrogate_on, + is_predicted, start->workload_injection_time, - end->next_packet_delay, start->travel_start_time, - end->travel_end_time, end->travel_end_time - start->travel_start_time, - start->is_there_another_pckt_in_queue); + start->travel_start_time, + end->travel_end_time, + latency); } // ==== START OF Surrogate functions definition ==== @@ -3408,6 +3521,14 @@ static void terminal_commit_packet_generate(terminal_state * s, tw_bf * bf, term // TODO (elkin): In the future, this ugly initialization could be done all in a single "line" instead of setting all values one by one. The reason to do it this way is because some old compilers do not understand other ways of initializing struct packet_sent sent; + unsigned int const feature_rail_id = dfdally_safe_rail_id(s, msg); + uint32_t const src_router_id = dfdally_terminal_src_router_id(s, msg); + uint32_t const src_group_id = src_router_id / s->params->num_routers; + uint32_t const dst_router_id = + dfdally_get_assigned_router_id_from_terminal( + s->params, msg->dfdally_dest_terminal_id, feature_rail_id); + uint32_t const dst_group_id = dst_router_id / s->params->num_routers; + sent.start.packet_ID = msg->packet_ID; sent.start.dest_terminal_lpid = msg->dest_terminal_lpid; sent.start.dfdally_dest_terminal_id = msg->dfdally_dest_terminal_id; @@ -3416,6 +3537,13 @@ static void terminal_commit_packet_generate(terminal_state * s, tw_bf * bf, term sent.start.processing_packet_delay = processing_packet_delay; sent.start.packet_size = msg->packet_size; sent.start.is_there_another_pckt_in_queue = msg->is_there_another_pckt_in_queue; + sent.start.caller_lp_gid = lp->gid; + sent.start.src_router_id = src_router_id; + sent.start.src_group_id = src_group_id; + sent.start.dst_router_id = dst_router_id; + sent.start.dst_group_id = dst_group_id; + sent.start.terminal_queue_length = dfdally_terminal_queue_length_bytes(s); + sent.start.terminal_vc_occupancy = dfdally_terminal_vc_occupancy_bytes(s); sent.next_packet_delay = -1; sent.message_data = msg_data; sent.remote_event_data = remote_data; @@ -3454,6 +3582,14 @@ static void terminal_dally_commit(terminal_state * s, case T_GENERATE: if(bf->c10) { // if the packet was sent as a prediction, store the prediction in memory assert(dally_surrogate_configured); + unsigned int const feature_rail_id = dfdally_safe_rail_id(s, msg); + uint32_t const src_router_id = dfdally_terminal_src_router_id(s, msg); + uint32_t const src_group_id = src_router_id / s->params->num_routers; + uint32_t const dst_router_id = + dfdally_get_assigned_router_id_from_terminal( + s->params, msg->dfdally_dest_terminal_id, feature_rail_id); + uint32_t const dst_group_id = dst_router_id / s->params->num_routers; + auto start = (struct packet_start) { .packet_ID = msg->packet_ID, .dest_terminal_lpid = msg->dest_terminal_lpid, @@ -3462,7 +3598,14 @@ static void terminal_dally_commit(terminal_state * s, .workload_injection_time = msg->msg_start_time, .processing_packet_delay = -1, .packet_size = msg->packet_size, - .is_there_another_pckt_in_queue = msg->is_there_another_pckt_in_queue + .is_there_another_pckt_in_queue = msg->is_there_another_pckt_in_queue, + + .src_router_id = src_router_id, + .src_group_id = src_group_id, + .dst_router_id = dst_router_id, + .dst_group_id = dst_group_id, + .terminal_queue_length = dfdally_terminal_queue_length_bytes(s), + .terminal_vc_occupancy = dfdally_terminal_vc_occupancy_bytes(s) }; // Saving @@ -4118,6 +4261,15 @@ static void packet_generate_predicted(terminal_state * s, tw_bf * bf, terminal_d // Using predictor to find latency double const processing_packet_delay = tw_now(lp) - s->last_in_queue_time; + + unsigned int const feature_rail_id = dfdally_safe_rail_id(s, msg); + uint32_t const src_router_id = dfdally_terminal_src_router_id(s, msg); + uint32_t const src_group_id = src_router_id / s->params->num_routers; + uint32_t const dst_router_id = + dfdally_get_assigned_router_id_from_terminal( + s->params, msg->dfdally_dest_terminal_id, feature_rail_id); + uint32_t const dst_group_id = dst_router_id / s->params->num_routers; + auto start = (struct packet_start) { .packet_ID = msg->packet_ID, .dest_terminal_lpid = msg->dest_terminal_lpid, @@ -4126,9 +4278,27 @@ static void packet_generate_predicted(terminal_state * s, tw_bf * bf, terminal_d .workload_injection_time = msg->msg_start_time, .processing_packet_delay = processing_packet_delay, .packet_size = msg->packet_size, - .is_there_another_pckt_in_queue = msg->is_there_another_pckt_in_queue + .is_there_another_pckt_in_queue = msg->is_there_another_pckt_in_queue, + + .src_router_id = src_router_id, + .src_group_id = src_group_id, + .dst_router_id = dst_router_id, + .dst_group_id = dst_group_id, + /* + * In predicted/surrogate generation, the high-fidelity terminal queue + * structures are bypassed, so dfdally_terminal_queue_length_bytes(s) + * can be zero even though the model was trained on the queue occupancy + * observed during high-fidelity packet generation. Avoid feeding an + * out-of-distribution zero into the LP-aware Torch-JIT model. + */ + .terminal_queue_length = dfdally_terminal_queue_length_bytes(s), + .terminal_vc_occupancy = dfdally_terminal_vc_occupancy_bytes(s) }; + if (start.terminal_queue_length == 0 && start.packet_size > 0) { + start.terminal_queue_length = start.packet_size; + } + struct packet_end const end = terminal_predictor->predict(s->predictor_data, lp, s->terminal_id, &start); double const latency = end.travel_end_time - start.travel_start_time; diff --git a/src/surrogate/init.c b/src/surrogate/init.c index 317f173a..e4bcaf46 100644 --- a/src/surrogate/init.c +++ b/src/surrogate/init.c @@ -98,9 +98,16 @@ bool network_surrogate_configure( char torch_jit_mode[MAX_NAME_LENGTH]; torch_jit_mode[0] = '\0'; configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_mode", anno, torch_jit_mode, MAX_NAME_LENGTH); - if (strcmp(torch_jit_mode, "single-static-model-for-all-terminals") != 0) { + + bool torch_jit_lp_aware_mode = false; + if (strcmp(torch_jit_mode, "single-static-model-for-all-terminals") == 0) { + torch_jit_lp_aware_mode = false; + } else if (strcmp(torch_jit_mode, "lp-aware-single-static-model") == 0) { + torch_jit_lp_aware_mode = true; + } else { tw_error(TW_LOC, "Unknown torch-jit mode `%s`", torch_jit_mode); } + surrogate_torch_set_lp_aware_mode(torch_jit_lp_aware_mode); char torch_jit_model_path[MAX_NAME_LENGTH]; torch_jit_model_path[0] = '\0'; diff --git a/src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py b/src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py new file mode 100644 index 00000000..0ad1252e --- /dev/null +++ b/src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py @@ -0,0 +1,77 @@ +from pathlib import Path +import pandas as pd + +indir = Path("packet-latency-trace") +files = sorted(indir.glob("packets-delay-gid=*.txt")) + +if not files: + raise SystemExit(f"No packet delay files found in {indir.resolve()}") + +cols = [ + "src_terminal", + "dest_terminal", + "packet_size", + "is_there_another_pckt_in_queue", + "caller_lp_gid", + "src_router_id", + "src_group_id", + "dst_router_id", + "dst_group_id", + "terminal_queue_length", + "terminal_vc_occupancy", + "processing_packet_delay", + "travel_end_time_delta", + "next_packet_delay", + "packet_id", + "is_surrogate_on", + "is_predicted", + "workload_injection", + "start", + "end", + "latency", +] + +frames = [] +for f in files: + df = pd.read_csv(f, comment="#", header=None) + + if df.shape[1] != len(cols): + raise SystemExit( + f"{f} has {df.shape[1]} columns, expected {len(cols)}. " + "This usually means the data-output patch was not applied, " + "or old packet-delay files are mixed with new ones." + ) + + df.columns = cols + df["source_file"] = f.name + frames.append(df) + +out = pd.concat(frames, ignore_index=True) + +# Keep only real high-fidelity rows. +out = out[(out["is_surrogate_on"] == 0) & (out["is_predicted"] == 0)] + +# Keep rows with usable targets. +out = out[out["travel_end_time_delta"] > 0] +out = out[out["next_packet_delay"] > 0] + +out_path = indir / "lp_aware_packet_latency_training.csv" +out.to_csv(out_path, index=False) + +print(f"Input files: {len(files)}") +print(f"Training rows: {len(out)}") +print(f"Wrote: {out_path}") + +print(df[[ + "src_terminal", + "dest_terminal", + "packet_size", + "caller_lp_gid", + "src_router_id", + "dst_router_id", + "terminal_queue_length", + "terminal_vc_occupancy", + "processing_packet_delay", + "travel_end_time_delta", + "next_packet_delay", +]].describe()) diff --git a/src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py b/src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py new file mode 100755 index 00000000..ecefe2cc --- /dev/null +++ b/src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +""" +Train an LP-aware packet-latency TorchScript model from CODES-generated CSV data. + +Expected input CSV: + ml_training_data/lp_aware_packet_latency_training.csv + +Input feature order must match the C++ Torch-JIT LP-aware mode: + 0 src_terminal + 1 dest_terminal + 2 packet_size + 3 is_there_another_pckt_in_queue + 4 caller_lp_gid + 5 src_router_id + 6 src_group_id + 7 dst_router_id + 8 dst_group_id + 9 terminal_queue_length + 10 terminal_vc_occupancy + 11 processing_packet_delay + +Targets: + 0 travel_end_time_delta + 1 next_packet_delay +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +from torch.utils.data import DataLoader, TensorDataset, random_split + + +FEATURE_COLUMNS = [ + "src_terminal", + "dest_terminal", + "packet_size", + "is_there_another_pckt_in_queue", + "caller_lp_gid", + "src_router_id", + "src_group_id", + "dst_router_id", + "dst_group_id", + "terminal_queue_length", + "terminal_vc_occupancy", + "processing_packet_delay", +] + +TARGET_COLUMNS = [ + "travel_end_time_delta", + "next_packet_delay", +] + + +class Standardizer: + def __init__(self) -> None: + self.mean: np.ndarray | None = None + self.std: np.ndarray | None = None + + def fit(self, x: np.ndarray) -> "Standardizer": + self.mean = x.mean(axis=0) + self.std = x.std(axis=0) + self.std[self.std < 1e-8] = 1.0 + return self + + def transform(self, x: np.ndarray) -> np.ndarray: + if self.mean is None or self.std is None: + raise RuntimeError("Standardizer must be fit before transform.") + return (x - self.mean) / self.std + + def inverse_transform(self, x: np.ndarray) -> np.ndarray: + if self.mean is None or self.std is None: + raise RuntimeError("Standardizer must be fit before inverse_transform.") + return x * self.std + self.mean + + def to_dict(self) -> dict: + if self.mean is None or self.std is None: + raise RuntimeError("Standardizer must be fit before serialization.") + return { + "mean": self.mean.tolist(), + "std": self.std.tolist(), + } + + +class LPAwarePacketLatencyModel(nn.Module): + def __init__( + self, + input_dim: int = len(FEATURE_COLUMNS), + output_dim: int = len(TARGET_COLUMNS), + hidden_dim: int = 128, + dropout: float = 0.0, + ) -> None: + super().__init__() + self.net = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, hidden_dim), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, output_dim), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.to(torch.float32) + return self.net(x) + + +class TorchScriptInferenceWrapper(nn.Module): + """ + Wrapper exported to C++. + + C++ passes raw 12-feature float32 vectors. This wrapper normalizes inputs, + runs the trained model, de-normalizes outputs, and clamps outputs to be + non-negative. + """ + + def __init__( + self, + model: nn.Module, + x_mean: torch.Tensor, + x_std: torch.Tensor, + y_mean: torch.Tensor, + y_std: torch.Tensor, + ) -> None: + super().__init__() + self.model = model + self.register_buffer("x_mean", x_mean) + self.register_buffer("x_std", x_std) + self.register_buffer("y_mean", y_mean) + self.register_buffer("y_std", y_std) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.to(torch.float32) + x_norm = (x - self.x_mean) / self.x_std + y_norm = self.model(x_norm) + y = y_norm * self.y_std + self.y_mean + return torch.clamp(y, min=0.0) + + +def load_training_data(csv_path: Path) -> tuple[np.ndarray, np.ndarray]: + if not csv_path.exists(): + raise FileNotFoundError(f"Training CSV does not exist: {csv_path}") + + df = pd.read_csv(csv_path) + + missing = [c for c in FEATURE_COLUMNS + TARGET_COLUMNS if c not in df.columns] + if missing: + raise ValueError(f"CSV is missing required columns: {missing}") + + df = df.replace([np.inf, -np.inf], np.nan) + before = len(df) + df = df.dropna(subset=FEATURE_COLUMNS + TARGET_COLUMNS) + after = len(df) + + if after == 0: + raise ValueError("No usable rows after dropping NaN/inf values.") + + # Keep only physically meaningful targets. + df = df[df["travel_end_time_delta"] > 0] + df = df[df["next_packet_delay"] > 0] + + if len(df) == 0: + raise ValueError("No rows with positive target values.") + + if len(df) < before: + print(f"Dropped {before - len(df)} unusable rows.") + + x = df[FEATURE_COLUMNS].to_numpy(dtype=np.float32) + y = df[TARGET_COLUMNS].to_numpy(dtype=np.float32) + + return x, y + + +def train(args: argparse.Namespace) -> None: + torch.manual_seed(args.seed) + np.random.seed(args.seed) + + x_raw, y_raw = load_training_data(args.csv) + + x_scaler = Standardizer().fit(x_raw) + y_scaler = Standardizer().fit(y_raw) + + x = x_scaler.transform(x_raw).astype(np.float32) + y = y_scaler.transform(y_raw).astype(np.float32) + + x_tensor = torch.from_numpy(x) + y_tensor = torch.from_numpy(y) + + dataset = TensorDataset(x_tensor, y_tensor) + + val_size = max(1, int(len(dataset) * args.val_fraction)) + train_size = len(dataset) - val_size + if train_size <= 0: + raise ValueError(f"Not enough rows for train/val split: {len(dataset)}") + + train_ds, val_ds = random_split( + dataset, + [train_size, val_size], + generator=torch.Generator().manual_seed(args.seed), + ) + + train_loader = DataLoader( + train_ds, + batch_size=args.batch_size, + shuffle=True, + drop_last=False, + ) + val_loader = DataLoader( + val_ds, + batch_size=args.batch_size, + shuffle=False, + drop_last=False, + ) + + device = torch.device(args.device) + model = LPAwarePacketLatencyModel( + hidden_dim=args.hidden_dim, + dropout=args.dropout, + ).to(device) + + optimizer = torch.optim.AdamW( + model.parameters(), + lr=args.lr, + weight_decay=args.weight_decay, + ) + loss_fn = nn.MSELoss() + + best_val = float("inf") + best_state: dict[str, torch.Tensor] | None = None + + for epoch in range(1, args.epochs + 1): + model.train() + train_losses = [] + + for xb, yb in train_loader: + xb = xb.to(device) + yb = yb.to(device) + + optimizer.zero_grad(set_to_none=True) + pred = model(xb) + loss = loss_fn(pred, yb) + loss.backward() + + if args.grad_clip > 0: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + + optimizer.step() + train_losses.append(loss.item()) + + model.eval() + val_losses = [] + with torch.no_grad(): + for xb, yb in val_loader: + xb = xb.to(device) + yb = yb.to(device) + pred = model(xb) + val_losses.append(loss_fn(pred, yb).item()) + + train_loss = float(np.mean(train_losses)) + val_loss = float(np.mean(val_losses)) + + if val_loss < best_val: + best_val = val_loss + best_state = { + k: v.detach().cpu().clone() + for k, v in model.state_dict().items() + } + + if epoch == 1 or epoch % args.log_every == 0 or epoch == args.epochs: + print( + f"epoch={epoch:04d} " + f"train_mse={train_loss:.6g} " + f"val_mse={val_loss:.6g} " + f"best_val_mse={best_val:.6g}" + ) + + if best_state is None: + raise RuntimeError("Training failed to produce a best model state.") + + model.load_state_dict(best_state) + model.eval() + + wrapper = TorchScriptInferenceWrapper( + model.cpu(), + x_mean=torch.tensor(x_scaler.mean, dtype=torch.float32).view(1, -1), + x_std=torch.tensor(x_scaler.std, dtype=torch.float32).view(1, -1), + y_mean=torch.tensor(y_scaler.mean, dtype=torch.float32).view(1, -1), + y_std=torch.tensor(y_scaler.std, dtype=torch.float32).view(1, -1), + ) + wrapper.eval() + + example = torch.zeros((1, len(FEATURE_COLUMNS)), dtype=torch.float32) + + with torch.no_grad(): + traced = torch.jit.trace(wrapper, example) + test_out = traced(example) + + if tuple(test_out.shape) != (1, len(TARGET_COLUMNS)): + raise RuntimeError(f"Unexpected traced output shape: {tuple(test_out.shape)}") + + args.output.parent.mkdir(parents=True, exist_ok=True) + traced.save(str(args.output)) + + metadata = { + "feature_columns": FEATURE_COLUMNS, + "target_columns": TARGET_COLUMNS, + "input_shape": [None, len(FEATURE_COLUMNS)], + "output_shape": [None, len(TARGET_COLUMNS)], + "input_dtype": "float32", + "output_dtype": "float32", + "x_scaler": x_scaler.to_dict(), + "y_scaler": y_scaler.to_dict(), + "rows": int(len(x_raw)), + "train_rows": int(train_size), + "val_rows": int(val_size), + "best_val_mse_scaled": best_val, + } + + metadata_path = args.output.with_suffix(args.output.suffix + ".metadata.json") + metadata_path.write_text(json.dumps(metadata, indent=2)) + + print(f"Saved TorchScript model: {args.output}") + print(f"Saved metadata: {metadata_path}") + print(f"Expected C++ input: float32 [batch, {len(FEATURE_COLUMNS)}]") + print(f"Expected C++ output: float32 [batch, {len(TARGET_COLUMNS)}]") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + + parser.add_argument( + "--csv", + type=Path, + default=Path("ml_training_data/lp_aware_packet_latency_training.csv"), + help="Training CSV generated from CODES packet latency logs.", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("ml_models/lp_aware_packet_latency_model.pt"), + help="Output TorchScript .pt path.", + ) + parser.add_argument("--epochs", type=int, default=50) + parser.add_argument("--batch-size", type=int, default=4096) + parser.add_argument("--hidden-dim", type=int, default=128) + parser.add_argument("--dropout", type=float, default=0.0) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--weight-decay", type=float, default=1e-5) + parser.add_argument("--val-fraction", type=float, default=0.1) + parser.add_argument("--grad-clip", type=float, default=1.0) + parser.add_argument("--seed", type=int, default=1234) + parser.add_argument("--log-every", type=int, default=5) + parser.add_argument( + "--device", + type=str, + default="cpu", + help="Use cpu for portable training/export; cuda is okay if available.", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + train(parse_args()) diff --git a/src/surrogate/ml_models/combine_packet_latency_traces_across_ranks.py b/src/surrogate/ml_models/director/combine_packet_latency_traces_across_ranks.py similarity index 100% rename from src/surrogate/ml_models/combine_packet_latency_traces_across_ranks.py rename to src/surrogate/ml_models/director/combine_packet_latency_traces_across_ranks.py diff --git a/src/surrogate/ml_models/director/lp_aware_packet_latency_model.py b/src/surrogate/ml_models/director/lp_aware_packet_latency_model.py new file mode 100644 index 00000000..b67c1463 --- /dev/null +++ b/src/surrogate/ml_models/director/lp_aware_packet_latency_model.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +LP-aware TorchScript packet-latency model. + +This is intentionally separate from the existing Torch-JIT model/exporter. +Use it only with: + + torch_jit_mode = lp-aware-single-static-model + +C++ feature order: + 0 src_terminal + 1 dfdally_dest_terminal_id + 2 packet_size + 3 is_there_another_pckt_in_queue + 4 caller_lp_gid + 5 src_router_id + 6 src_group_id + 7 dst_router_id + 8 dst_group_id + 9 terminal_queue_length + 10 terminal_vc_occupancy + 11 processing_packet_delay +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import torch +import torch.nn as nn + + +LP_AWARE_PACKET_FEATURES = [ + "src_terminal", + "dfdally_dest_terminal_id", + "packet_size", + "is_there_another_pckt_in_queue", + "caller_lp_gid", + "src_router_id", + "src_group_id", + "dst_router_id", + "dst_group_id", + "terminal_queue_length", + "terminal_vc_occupancy", + "processing_packet_delay", +] + +TORCH_PACKET_FEATURE_COUNT = len(LP_AWARE_PACKET_FEATURES) + + +class LPAwarePacketLatencyModel(nn.Module): + def __init__(self, hidden_dim: int = 64): + super().__init__() + self.net = nn.Sequential( + nn.Linear(TORCH_PACKET_FEATURE_COUNT, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, 2), + nn.Softplus(), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # C++ passes float32 [batch, 12]. + x = x.to(torch.float32) + return self.net(x) + + +def export_torchscript(output_path: Path, hidden_dim: int) -> None: + model = LPAwarePacketLatencyModel(hidden_dim=hidden_dim) + model.eval() + + example = torch.zeros((1, TORCH_PACKET_FEATURE_COUNT), dtype=torch.float32) + + with torch.no_grad(): + traced = torch.jit.trace(model, example) + + output_path.parent.mkdir(parents=True, exist_ok=True) + traced.save(str(output_path)) + print(f"saved LP-aware TorchScript model to {output_path}") + print(f"input shape: [batch, {TORCH_PACKET_FEATURE_COUNT}], dtype=float32") + print("output shape: [batch, 2]") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--output", + type=Path, + default=Path("lp_aware_packet_latency_model.pt"), + help="Path to write the TorchScript model.", + ) + parser.add_argument( + "--hidden-dim", + type=int, + default=64, + help="Hidden dimension for the prototype MLP.", + ) + args = parser.parse_args() + + export_torchscript(args.output, args.hidden_dim) + + +if __name__ == "__main__": + main() diff --git a/src/surrogate/ml_models/train_packet_latency_torchjit.py b/src/surrogate/ml_models/director/train_packet_latency_torchjit.py similarity index 100% rename from src/surrogate/ml_models/train_packet_latency_torchjit.py rename to src/surrogate/ml_models/director/train_packet_latency_torchjit.py diff --git a/src/surrogate/packet-latency-predictor/average.c b/src/surrogate/packet-latency-predictor/average.c index 4b14aedb..59b9e7d2 100644 --- a/src/surrogate/packet-latency-predictor/average.c +++ b/src/surrogate/packet-latency-predictor/average.c @@ -87,6 +87,22 @@ static struct packet_end predict_latency(struct latency_surrogate * data, tw_lp // TODO (Elkin): 10 is an arbitrary small value, but it should be nic_ts as implemented in `packet_getenerate` in dragonfly-dally double const next_packet_delay = data->aggregated_next_packet_delay.total_msgs == 0 ? 10 : data->aggregated_next_packet_delay.sum_latency / data->aggregated_next_packet_delay.total_msgs; + + static unsigned long long average_predict_calls = 0; + average_predict_calls++; + if (average_predict_calls <= 20 || average_predict_calls % 10000 == 0) { + fprintf(stderr, + "[average predict debug] calls=%llu latency=%f next_packet_delay=%f " + "next_delay_points=%u total_latency_points=%u dest_latency_points=%u\n", + average_predict_calls, + latency, + next_packet_delay, + data->aggregated_next_packet_delay.total_msgs, + data->aggregated_latency_for_all.total_msgs, + data->aggregated_latency[dest_terminal].total_msgs); + fflush(stderr); + } + return (struct packet_end) { .travel_end_time = packet_dest->travel_start_time + latency, .next_packet_delay = next_packet_delay, diff --git a/src/surrogate/packet-latency-predictor/torch-jit.C b/src/surrogate/packet-latency-predictor/torch-jit.C index d69a649d..77342609 100644 --- a/src/surrogate/packet-latency-predictor/torch-jit.C +++ b/src/surrogate/packet-latency-predictor/torch-jit.C @@ -6,12 +6,48 @@ #include #include #include +#include #include #include static torch::jit::Module packet_latency_model; +static bool torch_jit_lp_aware_mode = false; + +static constexpr int TORCH_JIT_LEGACY_FEATURE_COUNT = 4; +static constexpr int TORCH_JIT_LP_AWARE_FEATURE_COUNT = 12; + +void surrogate_torch_set_lp_aware_mode(bool enabled) { + torch_jit_lp_aware_mode = enabled; +} + +static std::vector build_lp_aware_packet_features( + tw_lp *lp, + unsigned int src_terminal, + struct packet_start const *packet_dest) +{ + assert(packet_dest != nullptr); + + return { + /* Existing four features first. */ + (float)src_terminal, + (float)packet_dest->dfdally_dest_terminal_id, + (float)packet_dest->packet_size, + packet_dest->is_there_another_pckt_in_queue ? 1.0f : 0.0f, + + /* Explicit LP/topology/context features. */ + lp ? (float)lp->gid : -1.0f, + (float)packet_dest->src_router_id, + (float)packet_dest->src_group_id, + (float)packet_dest->dst_router_id, + (float)packet_dest->dst_group_id, + (float)packet_dest->terminal_queue_length, + (float)packet_dest->terminal_vc_occupancy, + (float)packet_dest->processing_packet_delay + }; +} + inline void assert_correct_dims(at::Tensor * t) { int const dims = t->ndimension(); @@ -44,12 +80,24 @@ void surrogate_torch_init(char const * dir) { << std::endl; } - long int data_input[] = {0, 0, 0, 0}; - size_t const n_input = sizeof(data_input) / sizeof(long int); - std::vector inputs; torch::NoGradGuard no_grad; - inputs.emplace_back(torch::from_blob(data_input, {1, (int) n_input}, at::kLong)); + + if (torch_jit_lp_aware_mode) { + std::vector data_input(TORCH_JIT_LP_AWARE_FEATURE_COUNT, 0.0f); + inputs.emplace_back( + torch::from_blob( + data_input.data(), + {1, TORCH_JIT_LP_AWARE_FEATURE_COUNT}, + at::kFloat).clone()); + } else { + long int data_input[] = {0, 0, 0, 0}; + inputs.emplace_back( + torch::from_blob( + data_input, + {1, TORCH_JIT_LEGACY_FEATURE_COUNT}, + at::kLong).clone()); + } // Predicting value at::Tensor output = packet_latency_model.forward(inputs).toTensor(); @@ -63,24 +111,56 @@ static struct packet_end surrogate_torch_predict(void *, tw_lp * lp, unsigned in //auto t_start = std::chrono::high_resolution_clock::now(); // Create a vector of inputs. - long int data_input[] = { - src_terminal, - packet_dest->dfdally_dest_terminal_id, - packet_dest->packet_size, - packet_dest->is_there_another_pckt_in_queue - }; - size_t n_input = sizeof(data_input) / sizeof(long int); - std::vector inputs; - inputs.emplace_back(torch::from_blob(data_input, {1, (int) n_input}, at::kLong)); + + if (torch_jit_lp_aware_mode) { + std::vector data_input = + build_lp_aware_packet_features(lp, src_terminal, packet_dest); + + assert((int)data_input.size() == TORCH_JIT_LP_AWARE_FEATURE_COUNT); + + inputs.emplace_back( + torch::from_blob( + data_input.data(), + {1, TORCH_JIT_LP_AWARE_FEATURE_COUNT}, + at::kFloat).clone()); + } else { + long int data_input[] = { + src_terminal, + packet_dest->dfdally_dest_terminal_id, + packet_dest->packet_size, + packet_dest->is_there_another_pckt_in_queue + }; + + inputs.emplace_back( + torch::from_blob( + data_input, + {1, TORCH_JIT_LEGACY_FEATURE_COUNT}, + at::kLong).clone()); + } at::Tensor output = packet_latency_model.forward(inputs).toTensor(); //assert_correct_dims(&output); auto *out_data = output.data_ptr(); - return (struct packet_end) { - .travel_end_time = packet_dest->travel_start_time + (out_data[0] > 0 ? out_data[0] : 10), - .next_packet_delay = out_data[1] > 0 ? out_data[1] : 200, + double const raw_travel_delta = (double)out_data[0]; + double const raw_next_delay = (double)out_data[1]; + + double const min_travel_delta = 10.0; + double const min_next_packet_delay = 10.0; + + double const predicted_travel_delta = + std::isfinite(raw_travel_delta) && raw_travel_delta > min_travel_delta + ? raw_travel_delta + : min_travel_delta; + + double const predicted_next_packet_delay = + std::isfinite(raw_next_delay) && raw_next_delay > min_next_packet_delay + ? raw_next_delay + : min_next_packet_delay; +return (struct packet_end) { + .travel_end_time = packet_dest->travel_start_time + predicted_travel_delta, + .next_packet_delay = predicted_next_packet_delay, }; //auto t_end = std::chrono::high_resolution_clock::now(); From 5534e2ceb958ba249b4a41fe892cf5b01d631f93 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 27 May 2026 10:24:25 -0400 Subject: [PATCH 10/13] surrogate: add Torch-JIT debug diagnostics Add a NETWORK_SURROGATE debug_prints option for enabling diagnostic logging in the Torch-JIT packet latency predictor and Dragonfly Dally post-switch paths. Keep the diagnostics disabled by default so normal surrogate runs are not noisy, while preserving an explicit config knob for debugging inference-time feature construction and post-switch event behavior. --- .../packet-latency-predictor/torch-jit.h | 1 + .../tutorial-ping-pong-surrogate.conf.in | 3 + src/networks/model-net/dragonfly-dally.C | 57 ++++++++++++++ src/surrogate/init.c | 11 +++ .../packet-latency-predictor/torch-jit.C | 77 ++++++++++++++++++- 5 files changed, 148 insertions(+), 1 deletion(-) diff --git a/codes/surrogate/packet-latency-predictor/torch-jit.h b/codes/surrogate/packet-latency-predictor/torch-jit.h index cb1031b7..3a297319 100644 --- a/codes/surrogate/packet-latency-predictor/torch-jit.h +++ b/codes/surrogate/packet-latency-predictor/torch-jit.h @@ -10,6 +10,7 @@ extern "C" { #endif void surrogate_torch_set_lp_aware_mode(bool enabled); +void surrogate_torch_set_debug_prints(bool enabled); void surrogate_torch_init(char const * dir); extern struct packet_latency_predictor torch_latency_predictor; diff --git a/doc/example/tutorial-ping-pong-surrogate.conf.in b/doc/example/tutorial-ping-pong-surrogate.conf.in index bdf9be33..698d1e2d 100644 --- a/doc/example/tutorial-ping-pong-surrogate.conf.in +++ b/doc/example/tutorial-ping-pong-surrogate.conf.in @@ -79,6 +79,9 @@ NETWORK_SURROGATE { torch_jit_mode="${TORCH_JIT_MODE}"; torch_jit_model_path="${TORCH_JIT_MODEL_PATH}"; +# temporary surrogate debug prints. Options: 0 or 1 + debug_prints="${SURROGATE_DEBUG_PRINTS}"; + # selecting network treatment on switching to surrogate. Options: frezee, nothing network_treatment_on_switch="${NETWORK_TREATMENT}"; } diff --git a/src/networks/model-net/dragonfly-dally.C b/src/networks/model-net/dragonfly-dally.C index eba3ab1b..5d218e60 100644 --- a/src/networks/model-net/dragonfly-dally.C +++ b/src/networks/model-net/dragonfly-dally.C @@ -103,6 +103,11 @@ static long global_stalled_chunk_counter = 0; #define OUTPUT_SNAPSHOT 1 static int num_snapshots = 0; +static uint64_t dfdally_post_switch_terminal_events = 0; +static uint64_t dfdally_post_switch_router_events = 0; +static double dfdally_last_post_switch_terminal_now = -1.0; +static double dfdally_last_post_switch_router_now = -1.0; +static bool dfdally_surrogate_debug_prints = false; tw_stime * snapshot_times; char snapshot_filename[128]; @@ -2488,6 +2493,16 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params) enable_network_surrogate = (strcmp(enable_str, "1") == 0 || strcmp(enable_str, "true") == 0); } + char debug_prints_str[MAX_NAME_LENGTH]; + debug_prints_str[0] = '\0'; + configuration_get_value(&config, "NETWORK_SURROGATE", "debug_prints", anno, + debug_prints_str, MAX_NAME_LENGTH); + dfdally_surrogate_debug_prints = (strcmp(debug_prints_str, "1") == 0 || + strcmp(debug_prints_str, "true") == 0 || + strcmp(debug_prints_str, "TRUE") == 0 || + strcmp(debug_prints_str, "yes") == 0 || + strcmp(debug_prints_str, "YES") == 0); + // if surrogate mode has been set up if (enable_network_surrogate) { struct network_surrogate_config surr_conf = { @@ -7080,6 +7095,27 @@ terminal_dally_event( terminal_state * s, terminal_dally_message * msg, tw_lp * lp ) { + if (dfdally_surrogate_debug_prints && !is_dally_surrogate_on && tw_now(lp) >= 89000000.0) { + dfdally_post_switch_terminal_events++; + if (dfdally_post_switch_terminal_events <= 20 || + dfdally_post_switch_terminal_events % 100000 == 0) { + fprintf(stderr, + "[post-switch terminal debug] count=%llu now=%f lp=%llu type=%d " + "last_now=%f delta_now=%f\n", + (unsigned long long)dfdally_post_switch_terminal_events, + tw_now(lp), + (unsigned long long)lp->gid, + msg->type, + dfdally_last_post_switch_terminal_now, + dfdally_last_post_switch_terminal_now < 0.0 + ? -1.0 + : tw_now(lp) - dfdally_last_post_switch_terminal_now); + fflush(stderr); + } + dfdally_last_post_switch_terminal_now = tw_now(lp); + } + + msg->num_cll = 0; msg->num_rngs = 0; @@ -7150,6 +7186,27 @@ terminal_dally_event( terminal_state * s, static void router_dally_event(router_state * s, tw_bf * bf, terminal_dally_message * msg, tw_lp * lp) { + if (dfdally_surrogate_debug_prints && !is_dally_surrogate_on && tw_now(lp) >= 89000000.0) { + dfdally_post_switch_router_events++; + if (dfdally_post_switch_router_events <= 20 || + dfdally_post_switch_router_events % 100000 == 0) { + fprintf(stderr, + "[post-switch router debug] count=%llu now=%f lp=%llu type=%d " + "last_now=%f delta_now=%f\n", + (unsigned long long)dfdally_post_switch_router_events, + tw_now(lp), + (unsigned long long)lp->gid, + msg->type, + dfdally_last_post_switch_router_now, + dfdally_last_post_switch_router_now < 0.0 + ? -1.0 + : tw_now(lp) - dfdally_last_post_switch_router_now); + fflush(stderr); + } + dfdally_last_post_switch_router_now = tw_now(lp); + } + + msg->num_cll = 0; msg->num_rngs = 0; diff --git a/src/surrogate/init.c b/src/surrogate/init.c index e4bcaf46..c90eb05d 100644 --- a/src/surrogate/init.c +++ b/src/surrogate/init.c @@ -109,6 +109,17 @@ bool network_surrogate_configure( } surrogate_torch_set_lp_aware_mode(torch_jit_lp_aware_mode); + char debug_prints_str[MAX_NAME_LENGTH]; + debug_prints_str[0] = '\0'; + configuration_get_value(&config, "NETWORK_SURROGATE", "debug_prints", anno, + debug_prints_str, MAX_NAME_LENGTH); + bool debug_prints = (strcmp(debug_prints_str, "1") == 0 || + strcmp(debug_prints_str, "true") == 0 || + strcmp(debug_prints_str, "TRUE") == 0 || + strcmp(debug_prints_str, "yes") == 0 || + strcmp(debug_prints_str, "YES") == 0); + surrogate_torch_set_debug_prints(debug_prints); + char torch_jit_model_path[MAX_NAME_LENGTH]; torch_jit_model_path[0] = '\0'; configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_model_path", anno, torch_jit_model_path, MAX_NAME_LENGTH); diff --git a/src/surrogate/packet-latency-predictor/torch-jit.C b/src/surrogate/packet-latency-predictor/torch-jit.C index 77342609..2842a705 100644 --- a/src/surrogate/packet-latency-predictor/torch-jit.C +++ b/src/surrogate/packet-latency-predictor/torch-jit.C @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -13,7 +14,14 @@ static torch::jit::Module packet_latency_model; +static uint64_t torch_jit_predict_calls = 0; +static double torch_jit_forward_total_sec = 0.0; +static double torch_jit_min_travel_delta = 1.0e300; +static double torch_jit_max_travel_delta = -1.0e300; +static double torch_jit_min_next_delay = 1.0e300; +static double torch_jit_max_next_delay = -1.0e300; static bool torch_jit_lp_aware_mode = false; +static bool torch_jit_debug_prints = false; static constexpr int TORCH_JIT_LEGACY_FEATURE_COUNT = 4; static constexpr int TORCH_JIT_LP_AWARE_FEATURE_COUNT = 12; @@ -22,6 +30,10 @@ void surrogate_torch_set_lp_aware_mode(bool enabled) { torch_jit_lp_aware_mode = enabled; } +void surrogate_torch_set_debug_prints(bool enabled) { + torch_jit_debug_prints = enabled; +} + static std::vector build_lp_aware_packet_features( tw_lp *lp, unsigned int src_terminal, @@ -117,6 +129,28 @@ static struct packet_end surrogate_torch_predict(void *, tw_lp * lp, unsigned in std::vector data_input = build_lp_aware_packet_features(lp, src_terminal, packet_dest); + if (torch_jit_debug_prints && torch_jit_predict_calls < 20) { + fprintf(stderr, + "[torch-jit feature debug] " + "src_terminal=%g dest_terminal=%g packet_size=%g another=%g " + "caller_lp_gid=%g src_router_id=%g src_group_id=%g " + "dst_router_id=%g dst_group_id=%g terminal_queue_length=%g " + "terminal_vc_occupancy=%g processing_packet_delay=%g\n", + (double)data_input[0], + (double)data_input[1], + (double)data_input[2], + (double)data_input[3], + (double)data_input[4], + (double)data_input[5], + (double)data_input[6], + (double)data_input[7], + (double)data_input[8], + (double)data_input[9], + (double)data_input[10], + (double)data_input[11]); + fflush(stderr); + } + assert((int)data_input.size() == TORCH_JIT_LP_AWARE_FEATURE_COUNT); inputs.emplace_back( @@ -139,11 +173,17 @@ static struct packet_end surrogate_torch_predict(void *, tw_lp * lp, unsigned in at::kLong).clone()); } + auto const torch_jit_t0 = std::chrono::high_resolution_clock::now(); at::Tensor output = packet_latency_model.forward(inputs).toTensor(); + auto const torch_jit_t1 = std::chrono::high_resolution_clock::now(); + torch_jit_forward_total_sec += std::chrono::duration( + torch_jit_t1 - torch_jit_t0).count(); //assert_correct_dims(&output); auto *out_data = output.data_ptr(); - double const raw_travel_delta = (double)out_data[0]; + + torch_jit_predict_calls++; + double const raw_travel_delta = (double)out_data[0]; double const raw_next_delay = (double)out_data[1]; double const min_travel_delta = 10.0; @@ -158,6 +198,41 @@ static struct packet_end surrogate_torch_predict(void *, tw_lp * lp, unsigned in std::isfinite(raw_next_delay) && raw_next_delay > min_next_packet_delay ? raw_next_delay : min_next_packet_delay; + + if (raw_travel_delta < torch_jit_min_travel_delta) { + torch_jit_min_travel_delta = raw_travel_delta; + } + if (raw_travel_delta > torch_jit_max_travel_delta) { + torch_jit_max_travel_delta = raw_travel_delta; + } + if (raw_next_delay < torch_jit_min_next_delay) { + torch_jit_min_next_delay = raw_next_delay; + } + if (raw_next_delay > torch_jit_max_next_delay) { + torch_jit_max_next_delay = raw_next_delay; + } + + if (torch_jit_debug_prints && + (torch_jit_predict_calls <= 20 || + torch_jit_predict_calls % 10000 == 0)) { + fprintf(stderr, + "[torch-jit predict debug] calls=%llu avg_forward_us=%g " + "raw_travel_delta=%g raw_next_delay=%g " + "effective_travel_delta=%g effective_next_delay=%g " + "minmax_travel=[%g,%g] minmax_next=[%g,%g]\n", + (unsigned long long)torch_jit_predict_calls, + 1.0e6 * torch_jit_forward_total_sec / + (double)torch_jit_predict_calls, + raw_travel_delta, + raw_next_delay, + predicted_travel_delta, + predicted_next_packet_delay, + torch_jit_min_travel_delta, + torch_jit_max_travel_delta, + torch_jit_min_next_delay, + torch_jit_max_next_delay); + fflush(stderr); + } return (struct packet_end) { .travel_end_time = packet_dest->travel_start_time + predicted_travel_delta, .next_packet_delay = predicted_next_packet_delay, From 3c3f78dd68a3636b0830679b4855823d5eb69c0b Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 27 May 2026 13:28:41 -0400 Subject: [PATCH 11/13] Improve debug print check mechanism --- src/networks/model-net/dragonfly-dally.C | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/networks/model-net/dragonfly-dally.C b/src/networks/model-net/dragonfly-dally.C index 5d218e60..d22107e8 100644 --- a/src/networks/model-net/dragonfly-dally.C +++ b/src/networks/model-net/dragonfly-dally.C @@ -108,6 +108,7 @@ static uint64_t dfdally_post_switch_router_events = 0; static double dfdally_last_post_switch_terminal_now = -1.0; static double dfdally_last_post_switch_router_now = -1.0; static bool dfdally_surrogate_debug_prints = false; +static bool dfdally_surrogate_has_switched_once = false; tw_stime * snapshot_times; char snapshot_filename[128]; @@ -3169,6 +3170,7 @@ static inline void packet_latency_save_to_file( static void switch_surrogate(void) { is_dally_surrogate_on = ! is_dally_surrogate_on; + dfdally_surrogate_has_switched_once = true; } static bool is_surrogate_on_fun(void) { @@ -7095,7 +7097,7 @@ terminal_dally_event( terminal_state * s, terminal_dally_message * msg, tw_lp * lp ) { - if (dfdally_surrogate_debug_prints && !is_dally_surrogate_on && tw_now(lp) >= 89000000.0) { + if (dfdally_surrogate_debug_prints && dfdally_surrogate_has_switched_once && !is_dally_surrogate_on) { dfdally_post_switch_terminal_events++; if (dfdally_post_switch_terminal_events <= 20 || dfdally_post_switch_terminal_events % 100000 == 0) { @@ -7186,7 +7188,7 @@ terminal_dally_event( terminal_state * s, static void router_dally_event(router_state * s, tw_bf * bf, terminal_dally_message * msg, tw_lp * lp) { - if (dfdally_surrogate_debug_prints && !is_dally_surrogate_on && tw_now(lp) >= 89000000.0) { + if (dfdally_surrogate_debug_prints && dfdally_surrogate_has_switched_once && !is_dally_surrogate_on) { dfdally_post_switch_router_events++; if (dfdally_post_switch_router_events <= 20 || dfdally_post_switch_router_events % 100000 == 0) { From 399ec0f2ed9620025b5af71b4678c8881825f6fd Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 28 May 2026 14:55:18 -0400 Subject: [PATCH 12/13] Add LP-type-aware Torch-JIT surrogate models Extend the Torch-JIT surrogate path so different LP/component types can use separate ML models while preserving the existing single global packet-latency model for backward compatibility. Key changes: - Keep the existing packet_latency_model path for legacy Torch-JIT modes. - Add explicit torch_jit_mode support for lp-aware-lp-type-models. - Add terminal/default packet-latency model loading. - Add router timing model loading for queueing-delay prediction. - Add router timing trace output for high-fidelity PDES training runs. - Add configurable router timing trace stride to avoid excessive trace volume. - Correct router timing target to measure router-local residence/queueing delay from this_router_arrival instead of only propagation_ts. - Add component-level training scripts for terminal packet latency and router queueing delay TorchScript models. - Support CUDA training while exporting CPU-compatible TorchScript modules. - Wire debug_prints into the average packet-latency predictor so average debug output respects the config flag. The router model is intentionally timing-only: existing dragonfly routing still selects valid output ports, channels, and paths, while the ML model predicts only additional router-local queueing/contention delay. --- .../packet-latency-predictor/average.h | 2 + .../packet-latency-predictor/torch-jit.h | 26 ++ .../tutorial-ping-pong-surrogate.conf.in | 10 + doc/example/tutorial-ping-pong.conf.in | 2 + src/networks/model-net/dragonfly-dally.C | 180 +++++++- src/surrogate/init.c | 72 +++- ...bine_packet_latency_traces_across_ranks.py | 77 ---- .../train_lp_aware_packet_latency_model.py | 371 ---------------- .../train_torch_jit_packet_latency.py | 133 ++++++ .../train_torch_jit_router_timing.py | 204 +++++++++ .../packet-latency-predictor/average.c | 8 +- .../packet-latency-predictor/torch-jit.C | 397 ++++++++++++------ 12 files changed, 880 insertions(+), 602 deletions(-) delete mode 100644 src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py delete mode 100755 src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py create mode 100755 src/surrogate/ml_models/component_level/train_torch_jit_packet_latency.py create mode 100755 src/surrogate/ml_models/component_level/train_torch_jit_router_timing.py diff --git a/codes/surrogate/packet-latency-predictor/average.h b/codes/surrogate/packet-latency-predictor/average.h index f793bfa3..7c72eefd 100644 --- a/codes/surrogate/packet-latency-predictor/average.h +++ b/codes/surrogate/packet-latency-predictor/average.h @@ -18,6 +18,8 @@ extern "C" { extern double ignore_until; +void average_latency_predictor_set_debug_prints(int enabled); + struct packet_latency_predictor average_latency_predictor(int num_terminals); #ifdef __cplusplus diff --git a/codes/surrogate/packet-latency-predictor/torch-jit.h b/codes/surrogate/packet-latency-predictor/torch-jit.h index 3a297319..16fe94a1 100644 --- a/codes/surrogate/packet-latency-predictor/torch-jit.h +++ b/codes/surrogate/packet-latency-predictor/torch-jit.h @@ -13,6 +13,32 @@ void surrogate_torch_set_lp_aware_mode(bool enabled); void surrogate_torch_set_debug_prints(bool enabled); void surrogate_torch_init(char const * dir); +struct router_timing_prediction_start { + float router_id; + float group_id; + float output_port; + float output_chan; + float to_terminal; + float is_global; + float packet_size; + float chunk_size; + float output_vc_occupancy; + float output_queued_count; + float next_output_available_delta; + float nominal_router_delay; +}; + +void surrogate_torch_init_lp_type_models( + char const *terminal_model_path, + char const *router_timing_model_path, + char const *default_model_path); + +bool surrogate_torch_router_timing_model_enabled(void); + +double surrogate_torch_predict_router_queueing_delay( + struct router_timing_prediction_start const *start, + double fallback_queueing_delay); + extern struct packet_latency_predictor torch_latency_predictor; #ifdef __cplusplus diff --git a/doc/example/tutorial-ping-pong-surrogate.conf.in b/doc/example/tutorial-ping-pong-surrogate.conf.in index 698d1e2d..8040d921 100644 --- a/doc/example/tutorial-ping-pong-surrogate.conf.in +++ b/doc/example/tutorial-ping-pong-surrogate.conf.in @@ -55,6 +55,9 @@ PARAMS routing="prog-adaptive"; # folder path to store packet latency from terminal to terminal, if no value is given it won't save anything save_packet_latency_path="${PACKET_LATENCY_TRACE_PATH}"; +# folder path to store router-local timing rows for router queueing-delay training + save_router_timing_trace_path="${ROUTER_TIMING_TRACE_PATH}"; + save_router_timing_trace_stride="${ROUTER_TIMING_TRACE_STRIDE}"; # router buffer occupancy snapshots router_buffer_snapshots=( ${BUFFER_SNAPSHOTS} ); } @@ -76,8 +79,15 @@ NETWORK_SURROGATE { ignore_until="${IGNORE_UNTIL}"; # parameters for torch-jit latency predictor +# accepted modes: +# single-static-model-for-all-terminals: uses torch_jit_model_path, input [1,4], output [1,2] +# lp-aware-single-static-model: uses torch_jit_model_path, input [1,12], output [1,2] +# lp-aware-lp-type-models: uses the terminal/router/default paths below torch_jit_mode="${TORCH_JIT_MODE}"; torch_jit_model_path="${TORCH_JIT_MODEL_PATH}"; + torch_jit_terminal_model_path="${TORCH_JIT_TERMINAL_MODEL_PATH}"; + torch_jit_router_timing_model_path="${TORCH_JIT_ROUTER_TIMING_MODEL_PATH}"; + torch_jit_default_model_path="${TORCH_JIT_DEFAULT_MODEL_PATH}"; # temporary surrogate debug prints. Options: 0 or 1 debug_prints="${SURROGATE_DEBUG_PRINTS}"; diff --git a/doc/example/tutorial-ping-pong.conf.in b/doc/example/tutorial-ping-pong.conf.in index e8e2ce4e..042ee417 100644 --- a/doc/example/tutorial-ping-pong.conf.in +++ b/doc/example/tutorial-ping-pong.conf.in @@ -54,4 +54,6 @@ PARAMS router_buffer_snapshots=( ${BUFFER_SNAPSHOTS} ); # folder path to store packet latency from terminal to terminal, if no value is given it won't save anything save_packet_latency_path="${PACKET_LATENCY_TRACE_PATH}"; + save_router_timing_trace_path="${ROUTER_TIMING_TRACE_PATH}"; + save_router_timing_trace_stride="${ROUTER_TIMING_TRACE_STRIDE}"; } diff --git a/src/networks/model-net/dragonfly-dally.C b/src/networks/model-net/dragonfly-dally.C index d22107e8..cefff9c0 100644 --- a/src/networks/model-net/dragonfly-dally.C +++ b/src/networks/model-net/dragonfly-dally.C @@ -22,6 +22,9 @@ #include "codes/model-net-method.h" #include "codes/model-net-lp.h" #include "codes/surrogate/init.h" +#ifdef USE_TORCH +#include "codes/surrogate/packet-latency-predictor/torch-jit.h" +#endif #include "codes/net/dragonfly-dally.h" #include "quicklist.h" #include "sys/file.h" @@ -199,6 +202,12 @@ static char router_sample_file[MAX_NAME_LENGTH]; static FILE * packet_latency_f = NULL; static void setup_packet_latency_path(char const * const dir_to_save); +// File to store router-local timing rows for training router queueing-delay models. +static FILE * router_timing_trace_f = NULL; +static int router_timing_trace_stride = 1; +static unsigned long long router_timing_trace_counter = 0; +static void setup_router_timing_trace_path(char const * const dir_to_save); + // ==== START OF Parameters to tune surrogate mode ==== // @@ -2485,6 +2494,30 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params) setup_packet_latency_path(packet_latency_path); } + // Router timing trace path to store pure-PDES router-local queueing-delay training rows. + char router_timing_trace_path[MAX_NAME_LENGTH]; + router_timing_trace_path[0] = '\0'; + configuration_get_value(&config, "PARAMS", "save_router_timing_trace_path", anno, + router_timing_trace_path, MAX_NAME_LENGTH); + if(strlen(router_timing_trace_path) > 0) { + setup_router_timing_trace_path(router_timing_trace_path); + } + + char router_timing_trace_stride_str[MAX_NAME_LENGTH]; + router_timing_trace_stride_str[0] = '\0'; + configuration_get_value(&config, "PARAMS", "save_router_timing_trace_stride", anno, + router_timing_trace_stride_str, MAX_NAME_LENGTH); + if(strlen(router_timing_trace_stride_str) > 0) { + int const parsed_stride = atoi(router_timing_trace_stride_str); + if(parsed_stride > 0) { + router_timing_trace_stride = parsed_stride; + } else { + tw_warning(TW_LOC, + "Ignoring invalid save_router_timing_trace_stride=%s; using stride=1", + router_timing_trace_stride_str); + } + } + // START Surrogate configuration char enable_str[MAX_NAME_LENGTH]; enable_str[0] = '\0'; @@ -2598,6 +2631,84 @@ static void setup_packet_latency_path(char const * const dir_to_save) { "packet_id,is_surrogate_on,is_predicted,workload_injection,start,end,latency\n"); } + +static void setup_router_timing_trace_path(char const * const dir_to_save) { + assert(router_timing_trace_f == NULL); + int const NO_ERROR = 0; + struct stat st; + memset(&st, 0, sizeof(struct stat)); + if(g_tw_mynode == 0 && stat(dir_to_save, &st) == -1) { + int res = mkdir(dir_to_save, 0700); + if (res != NO_ERROR) { + tw_error(TW_LOC, "Error (%d) occurred when attempting to mkdir folder `%s`", errno, dir_to_save); + } + } + MPI_Barrier(MPI_COMM_CODES); + + char const fmt[] = "%s/router-timing-gid=%lu.txt"; + int sz = snprintf(NULL, 0, fmt, dir_to_save, g_tw_mynode); + char filename_path[sz + 1]; + snprintf(filename_path, sizeof(filename_path), fmt, dir_to_save, g_tw_mynode); + router_timing_trace_f = fopen(filename_path, "w+"); + if(!router_timing_trace_f) { + tw_error(TW_LOC, "File %s could not be opened", filename_path); + } + + fprintf(router_timing_trace_f, + "#router_id,group_id,output_port,output_chan,to_terminal,is_global," + "packet_size,chunk_size,output_vc_occupancy,output_queued_count," + "next_output_available_delta,nominal_router_delay,router_queueing_delay," + "actual_router_delay,is_surrogate_on,is_predicted,now,packet_id,src_terminal,dest_terminal,next_stop\n"); +} + +static inline void router_timing_trace_save_to_file( + router_state const *s, + terminal_dally_message const *pkt, + int output_port, + int output_chan, + int to_terminal, + int is_global, + double next_output_available_delta, + double nominal_router_delay, + double actual_router_delay, + bool surrogate_on, + bool is_predicted, + tw_lp *lp) +{ + if (!router_timing_trace_f) { return; } + + unsigned long long const trace_idx = router_timing_trace_counter++; + if(router_timing_trace_stride > 1 && + (trace_idx % (unsigned long long)router_timing_trace_stride) != 0ULL) { + return; + } + + double const router_queueing_delay = maxd(0.0, actual_router_delay - nominal_router_delay); + fprintf(router_timing_trace_f, + "%u,%d,%d,%d,%d,%d,%u,%d,%d,%d,%f,%f,%f,%f,%d,%d,%f,%lu,%u,%u,%llu\n", + s->router_id, + s->group_id, + output_port, + output_chan, + to_terminal, + is_global, + pkt->packet_size, + s->params->chunk_size, + s->vc_occupancy[output_port][output_chan], + s->queued_count[output_port], + next_output_available_delta, + nominal_router_delay, + router_queueing_delay, + actual_router_delay, + surrogate_on, + is_predicted, + tw_now(lp), + (unsigned long)pkt->packet_ID, + pkt->dfdally_src_terminal_id, + pkt->dfdally_dest_terminal_id, + (unsigned long long)pkt->next_stop); +} + /* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */ void dragonfly_dally_report_stats() { @@ -2632,6 +2743,11 @@ void dragonfly_dally_report_stats() if (packet_latency_f) { fclose(packet_latency_f); + packet_latency_f = NULL; + } + if (router_timing_trace_f) { + fclose(router_timing_trace_f); + router_timing_trace_f = NULL; } /* print statistics */ if(!g_tw_mynode) @@ -6836,14 +6952,76 @@ static void router_packet_send( router_state * s, tw_bf * bf, terminal_dally_mes injection_delay += s->params->router_delay; + double const next_output_available_delta = + maxd(0.0, s->next_output_available_time[output_port] - tw_now(lp)); + double const nominal_router_delay = injection_delay + propagation_delay; + msg->saved_available_time = s->next_output_available_time[output_port]; - s->next_output_available_time[output_port] = + s->next_output_available_time[output_port] = maxd(s->next_output_available_time[output_port], tw_now(lp)); s->next_output_available_time[output_port] += injection_delay; injection_ts = s->next_output_available_time[output_port] - tw_now(lp); propagation_ts = injection_ts + propagation_delay; + /* + * High-fidelity router-local delay target for ML training. + * + * propagation_ts is only the delay from this router send event to the next + * hop. It does not include the time the chunk already spent waiting in this + * router's queue before router_packet_send() fired. For the router timing + * model, the useful target is the total local residence+service+propagation + * time from this_router_arrival through arrival at the next hop. + * + * nominal_router_delay is the deterministic no-queueing service+propagation + * delay. Therefore: + * + * queueing_delay = max(0, highdef_actual_router_delay - nominal_router_delay) + * + * captures router-local waiting/contention time. + */ + double const highdef_actual_router_delay = + maxd(0.0, s->next_output_available_time[output_port] - + cur_entry->msg.this_router_arrival) + + propagation_delay; + bool router_timing_prediction_used = false; +#ifdef USE_TORCH + if (is_dally_surrogate_on && surrogate_torch_router_timing_model_enabled()) { + struct router_timing_prediction_start timing_start = { + .router_id = (float)s->router_id, + .group_id = (float)s->group_id, + .output_port = (float)output_port, + .output_chan = (float)output_chan, + .to_terminal = (float)to_terminal, + .is_global = (float)global, + .packet_size = (float)cur_entry->msg.packet_size, + .chunk_size = (float)s->params->chunk_size, + .output_vc_occupancy = (float)s->vc_occupancy[output_port][output_chan], + .output_queued_count = (float)s->queued_count[output_port], + .next_output_available_delta = (float)next_output_available_delta, + .nominal_router_delay = (float)nominal_router_delay, + }; + double const predicted_queueing_delay = + surrogate_torch_predict_router_queueing_delay(&timing_start, 0.0); + propagation_ts = nominal_router_delay + predicted_queueing_delay; + router_timing_prediction_used = true; + } +#endif + + router_timing_trace_save_to_file( + s, + &cur_entry->msg, + output_port, + output_chan, + to_terminal, + global, + next_output_available_delta, + nominal_router_delay, + highdef_actual_router_delay, + is_dally_surrogate_on, + router_timing_prediction_used, + lp); + cur_entry->msg.this_router_ptp_latency = s->next_output_available_time[output_port] - cur_entry->msg.this_router_arrival; msg->this_router_ptp_latency = cur_entry->msg.this_router_ptp_latency; diff --git a/src/surrogate/init.c b/src/surrogate/init.c index c90eb05d..1e680a15 100644 --- a/src/surrogate/init.c +++ b/src/surrogate/init.c @@ -85,11 +85,22 @@ bool network_surrogate_configure( } // Determining which predictor to set up and return + char debug_prints_str[MAX_NAME_LENGTH]; + debug_prints_str[0] = '\0'; + configuration_get_value(&config, "NETWORK_SURROGATE", "debug_prints", anno, + debug_prints_str, MAX_NAME_LENGTH); + bool debug_prints = (strcmp(debug_prints_str, "1") == 0 || + strcmp(debug_prints_str, "true") == 0 || + strcmp(debug_prints_str, "TRUE") == 0 || + strcmp(debug_prints_str, "yes") == 0 || + strcmp(debug_prints_str, "YES") == 0); + char latency_pred_name[MAX_NAME_LENGTH]; latency_pred_name[0] = '\0'; configuration_get_value(&config, "NETWORK_SURROGATE", "packet_latency_predictor", anno, latency_pred_name, MAX_NAME_LENGTH); if (*latency_pred_name) { if (strcmp(latency_pred_name, "average") == 0) { + average_latency_predictor_set_debug_prints(debug_prints); current_net_predictor = average_latency_predictor(sc->total_terminals); *pl_pred = ¤t_net_predictor; @@ -99,31 +110,49 @@ bool network_surrogate_configure( torch_jit_mode[0] = '\0'; configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_mode", anno, torch_jit_mode, MAX_NAME_LENGTH); - bool torch_jit_lp_aware_mode = false; + surrogate_torch_set_debug_prints(debug_prints); + if (strcmp(torch_jit_mode, "single-static-model-for-all-terminals") == 0) { - torch_jit_lp_aware_mode = false; + surrogate_torch_set_lp_aware_mode(false); + + char torch_jit_model_path[MAX_NAME_LENGTH]; + torch_jit_model_path[0] = '\0'; + configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_model_path", anno, + torch_jit_model_path, MAX_NAME_LENGTH); + surrogate_torch_init(torch_jit_model_path); } else if (strcmp(torch_jit_mode, "lp-aware-single-static-model") == 0) { - torch_jit_lp_aware_mode = true; + surrogate_torch_set_lp_aware_mode(true); + + char torch_jit_model_path[MAX_NAME_LENGTH]; + torch_jit_model_path[0] = '\0'; + configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_model_path", anno, + torch_jit_model_path, MAX_NAME_LENGTH); + surrogate_torch_init(torch_jit_model_path); + } else if (strcmp(torch_jit_mode, "lp-aware-lp-type-models") == 0) { + char terminal_model_path[MAX_NAME_LENGTH]; + char router_timing_model_path[MAX_NAME_LENGTH]; + char default_model_path[MAX_NAME_LENGTH]; + terminal_model_path[0] = '\0'; + router_timing_model_path[0] = '\0'; + default_model_path[0] = '\0'; + + configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_terminal_model_path", anno, + terminal_model_path, MAX_NAME_LENGTH); + configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_router_timing_model_path", anno, + router_timing_model_path, MAX_NAME_LENGTH); + configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_default_model_path", anno, + default_model_path, MAX_NAME_LENGTH); + + surrogate_torch_init_lp_type_models( + terminal_model_path, + router_timing_model_path, + default_model_path); } else { - tw_error(TW_LOC, "Unknown torch-jit mode `%s`", torch_jit_mode); + tw_error(TW_LOC, + "Unknown torch-jit mode `%s` (expected single-static-model-for-all-terminals, " + "lp-aware-single-static-model, or lp-aware-lp-type-models)", + torch_jit_mode); } - surrogate_torch_set_lp_aware_mode(torch_jit_lp_aware_mode); - - char debug_prints_str[MAX_NAME_LENGTH]; - debug_prints_str[0] = '\0'; - configuration_get_value(&config, "NETWORK_SURROGATE", "debug_prints", anno, - debug_prints_str, MAX_NAME_LENGTH); - bool debug_prints = (strcmp(debug_prints_str, "1") == 0 || - strcmp(debug_prints_str, "true") == 0 || - strcmp(debug_prints_str, "TRUE") == 0 || - strcmp(debug_prints_str, "yes") == 0 || - strcmp(debug_prints_str, "YES") == 0); - surrogate_torch_set_debug_prints(debug_prints); - - char torch_jit_model_path[MAX_NAME_LENGTH]; - torch_jit_model_path[0] = '\0'; - configuration_get_value(&config, "NETWORK_SURROGATE", "torch_jit_model_path", anno, torch_jit_model_path, MAX_NAME_LENGTH); - surrogate_torch_init(torch_jit_model_path); *pl_pred = &torch_latency_predictor; #endif @@ -137,6 +166,7 @@ bool network_surrogate_configure( ")", latency_pred_name); } } else { + average_latency_predictor_set_debug_prints(debug_prints); current_net_predictor = average_latency_predictor(sc->total_terminals); *pl_pred = ¤t_net_predictor; master_printf("Enabling average packet latency predictor (default behaviour)\n"); diff --git a/src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py b/src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py deleted file mode 100644 index 0ad1252e..00000000 --- a/src/surrogate/ml_models/component_level/combine_packet_latency_traces_across_ranks.py +++ /dev/null @@ -1,77 +0,0 @@ -from pathlib import Path -import pandas as pd - -indir = Path("packet-latency-trace") -files = sorted(indir.glob("packets-delay-gid=*.txt")) - -if not files: - raise SystemExit(f"No packet delay files found in {indir.resolve()}") - -cols = [ - "src_terminal", - "dest_terminal", - "packet_size", - "is_there_another_pckt_in_queue", - "caller_lp_gid", - "src_router_id", - "src_group_id", - "dst_router_id", - "dst_group_id", - "terminal_queue_length", - "terminal_vc_occupancy", - "processing_packet_delay", - "travel_end_time_delta", - "next_packet_delay", - "packet_id", - "is_surrogate_on", - "is_predicted", - "workload_injection", - "start", - "end", - "latency", -] - -frames = [] -for f in files: - df = pd.read_csv(f, comment="#", header=None) - - if df.shape[1] != len(cols): - raise SystemExit( - f"{f} has {df.shape[1]} columns, expected {len(cols)}. " - "This usually means the data-output patch was not applied, " - "or old packet-delay files are mixed with new ones." - ) - - df.columns = cols - df["source_file"] = f.name - frames.append(df) - -out = pd.concat(frames, ignore_index=True) - -# Keep only real high-fidelity rows. -out = out[(out["is_surrogate_on"] == 0) & (out["is_predicted"] == 0)] - -# Keep rows with usable targets. -out = out[out["travel_end_time_delta"] > 0] -out = out[out["next_packet_delay"] > 0] - -out_path = indir / "lp_aware_packet_latency_training.csv" -out.to_csv(out_path, index=False) - -print(f"Input files: {len(files)}") -print(f"Training rows: {len(out)}") -print(f"Wrote: {out_path}") - -print(df[[ - "src_terminal", - "dest_terminal", - "packet_size", - "caller_lp_gid", - "src_router_id", - "dst_router_id", - "terminal_queue_length", - "terminal_vc_occupancy", - "processing_packet_delay", - "travel_end_time_delta", - "next_packet_delay", -]].describe()) diff --git a/src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py b/src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py deleted file mode 100755 index ecefe2cc..00000000 --- a/src/surrogate/ml_models/component_level/train_lp_aware_packet_latency_model.py +++ /dev/null @@ -1,371 +0,0 @@ -#!/usr/bin/env python3 -""" -Train an LP-aware packet-latency TorchScript model from CODES-generated CSV data. - -Expected input CSV: - ml_training_data/lp_aware_packet_latency_training.csv - -Input feature order must match the C++ Torch-JIT LP-aware mode: - 0 src_terminal - 1 dest_terminal - 2 packet_size - 3 is_there_another_pckt_in_queue - 4 caller_lp_gid - 5 src_router_id - 6 src_group_id - 7 dst_router_id - 8 dst_group_id - 9 terminal_queue_length - 10 terminal_vc_occupancy - 11 processing_packet_delay - -Targets: - 0 travel_end_time_delta - 1 next_packet_delay -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -import numpy as np -import pandas as pd -import torch -import torch.nn as nn -from torch.utils.data import DataLoader, TensorDataset, random_split - - -FEATURE_COLUMNS = [ - "src_terminal", - "dest_terminal", - "packet_size", - "is_there_another_pckt_in_queue", - "caller_lp_gid", - "src_router_id", - "src_group_id", - "dst_router_id", - "dst_group_id", - "terminal_queue_length", - "terminal_vc_occupancy", - "processing_packet_delay", -] - -TARGET_COLUMNS = [ - "travel_end_time_delta", - "next_packet_delay", -] - - -class Standardizer: - def __init__(self) -> None: - self.mean: np.ndarray | None = None - self.std: np.ndarray | None = None - - def fit(self, x: np.ndarray) -> "Standardizer": - self.mean = x.mean(axis=0) - self.std = x.std(axis=0) - self.std[self.std < 1e-8] = 1.0 - return self - - def transform(self, x: np.ndarray) -> np.ndarray: - if self.mean is None or self.std is None: - raise RuntimeError("Standardizer must be fit before transform.") - return (x - self.mean) / self.std - - def inverse_transform(self, x: np.ndarray) -> np.ndarray: - if self.mean is None or self.std is None: - raise RuntimeError("Standardizer must be fit before inverse_transform.") - return x * self.std + self.mean - - def to_dict(self) -> dict: - if self.mean is None or self.std is None: - raise RuntimeError("Standardizer must be fit before serialization.") - return { - "mean": self.mean.tolist(), - "std": self.std.tolist(), - } - - -class LPAwarePacketLatencyModel(nn.Module): - def __init__( - self, - input_dim: int = len(FEATURE_COLUMNS), - output_dim: int = len(TARGET_COLUMNS), - hidden_dim: int = 128, - dropout: float = 0.0, - ) -> None: - super().__init__() - self.net = nn.Sequential( - nn.Linear(input_dim, hidden_dim), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(hidden_dim, hidden_dim), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(hidden_dim, output_dim), - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = x.to(torch.float32) - return self.net(x) - - -class TorchScriptInferenceWrapper(nn.Module): - """ - Wrapper exported to C++. - - C++ passes raw 12-feature float32 vectors. This wrapper normalizes inputs, - runs the trained model, de-normalizes outputs, and clamps outputs to be - non-negative. - """ - - def __init__( - self, - model: nn.Module, - x_mean: torch.Tensor, - x_std: torch.Tensor, - y_mean: torch.Tensor, - y_std: torch.Tensor, - ) -> None: - super().__init__() - self.model = model - self.register_buffer("x_mean", x_mean) - self.register_buffer("x_std", x_std) - self.register_buffer("y_mean", y_mean) - self.register_buffer("y_std", y_std) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = x.to(torch.float32) - x_norm = (x - self.x_mean) / self.x_std - y_norm = self.model(x_norm) - y = y_norm * self.y_std + self.y_mean - return torch.clamp(y, min=0.0) - - -def load_training_data(csv_path: Path) -> tuple[np.ndarray, np.ndarray]: - if not csv_path.exists(): - raise FileNotFoundError(f"Training CSV does not exist: {csv_path}") - - df = pd.read_csv(csv_path) - - missing = [c for c in FEATURE_COLUMNS + TARGET_COLUMNS if c not in df.columns] - if missing: - raise ValueError(f"CSV is missing required columns: {missing}") - - df = df.replace([np.inf, -np.inf], np.nan) - before = len(df) - df = df.dropna(subset=FEATURE_COLUMNS + TARGET_COLUMNS) - after = len(df) - - if after == 0: - raise ValueError("No usable rows after dropping NaN/inf values.") - - # Keep only physically meaningful targets. - df = df[df["travel_end_time_delta"] > 0] - df = df[df["next_packet_delay"] > 0] - - if len(df) == 0: - raise ValueError("No rows with positive target values.") - - if len(df) < before: - print(f"Dropped {before - len(df)} unusable rows.") - - x = df[FEATURE_COLUMNS].to_numpy(dtype=np.float32) - y = df[TARGET_COLUMNS].to_numpy(dtype=np.float32) - - return x, y - - -def train(args: argparse.Namespace) -> None: - torch.manual_seed(args.seed) - np.random.seed(args.seed) - - x_raw, y_raw = load_training_data(args.csv) - - x_scaler = Standardizer().fit(x_raw) - y_scaler = Standardizer().fit(y_raw) - - x = x_scaler.transform(x_raw).astype(np.float32) - y = y_scaler.transform(y_raw).astype(np.float32) - - x_tensor = torch.from_numpy(x) - y_tensor = torch.from_numpy(y) - - dataset = TensorDataset(x_tensor, y_tensor) - - val_size = max(1, int(len(dataset) * args.val_fraction)) - train_size = len(dataset) - val_size - if train_size <= 0: - raise ValueError(f"Not enough rows for train/val split: {len(dataset)}") - - train_ds, val_ds = random_split( - dataset, - [train_size, val_size], - generator=torch.Generator().manual_seed(args.seed), - ) - - train_loader = DataLoader( - train_ds, - batch_size=args.batch_size, - shuffle=True, - drop_last=False, - ) - val_loader = DataLoader( - val_ds, - batch_size=args.batch_size, - shuffle=False, - drop_last=False, - ) - - device = torch.device(args.device) - model = LPAwarePacketLatencyModel( - hidden_dim=args.hidden_dim, - dropout=args.dropout, - ).to(device) - - optimizer = torch.optim.AdamW( - model.parameters(), - lr=args.lr, - weight_decay=args.weight_decay, - ) - loss_fn = nn.MSELoss() - - best_val = float("inf") - best_state: dict[str, torch.Tensor] | None = None - - for epoch in range(1, args.epochs + 1): - model.train() - train_losses = [] - - for xb, yb in train_loader: - xb = xb.to(device) - yb = yb.to(device) - - optimizer.zero_grad(set_to_none=True) - pred = model(xb) - loss = loss_fn(pred, yb) - loss.backward() - - if args.grad_clip > 0: - torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) - - optimizer.step() - train_losses.append(loss.item()) - - model.eval() - val_losses = [] - with torch.no_grad(): - for xb, yb in val_loader: - xb = xb.to(device) - yb = yb.to(device) - pred = model(xb) - val_losses.append(loss_fn(pred, yb).item()) - - train_loss = float(np.mean(train_losses)) - val_loss = float(np.mean(val_losses)) - - if val_loss < best_val: - best_val = val_loss - best_state = { - k: v.detach().cpu().clone() - for k, v in model.state_dict().items() - } - - if epoch == 1 or epoch % args.log_every == 0 or epoch == args.epochs: - print( - f"epoch={epoch:04d} " - f"train_mse={train_loss:.6g} " - f"val_mse={val_loss:.6g} " - f"best_val_mse={best_val:.6g}" - ) - - if best_state is None: - raise RuntimeError("Training failed to produce a best model state.") - - model.load_state_dict(best_state) - model.eval() - - wrapper = TorchScriptInferenceWrapper( - model.cpu(), - x_mean=torch.tensor(x_scaler.mean, dtype=torch.float32).view(1, -1), - x_std=torch.tensor(x_scaler.std, dtype=torch.float32).view(1, -1), - y_mean=torch.tensor(y_scaler.mean, dtype=torch.float32).view(1, -1), - y_std=torch.tensor(y_scaler.std, dtype=torch.float32).view(1, -1), - ) - wrapper.eval() - - example = torch.zeros((1, len(FEATURE_COLUMNS)), dtype=torch.float32) - - with torch.no_grad(): - traced = torch.jit.trace(wrapper, example) - test_out = traced(example) - - if tuple(test_out.shape) != (1, len(TARGET_COLUMNS)): - raise RuntimeError(f"Unexpected traced output shape: {tuple(test_out.shape)}") - - args.output.parent.mkdir(parents=True, exist_ok=True) - traced.save(str(args.output)) - - metadata = { - "feature_columns": FEATURE_COLUMNS, - "target_columns": TARGET_COLUMNS, - "input_shape": [None, len(FEATURE_COLUMNS)], - "output_shape": [None, len(TARGET_COLUMNS)], - "input_dtype": "float32", - "output_dtype": "float32", - "x_scaler": x_scaler.to_dict(), - "y_scaler": y_scaler.to_dict(), - "rows": int(len(x_raw)), - "train_rows": int(train_size), - "val_rows": int(val_size), - "best_val_mse_scaled": best_val, - } - - metadata_path = args.output.with_suffix(args.output.suffix + ".metadata.json") - metadata_path.write_text(json.dumps(metadata, indent=2)) - - print(f"Saved TorchScript model: {args.output}") - print(f"Saved metadata: {metadata_path}") - print(f"Expected C++ input: float32 [batch, {len(FEATURE_COLUMNS)}]") - print(f"Expected C++ output: float32 [batch, {len(TARGET_COLUMNS)}]") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - - parser.add_argument( - "--csv", - type=Path, - default=Path("ml_training_data/lp_aware_packet_latency_training.csv"), - help="Training CSV generated from CODES packet latency logs.", - ) - parser.add_argument( - "--output", - type=Path, - default=Path("ml_models/lp_aware_packet_latency_model.pt"), - help="Output TorchScript .pt path.", - ) - parser.add_argument("--epochs", type=int, default=50) - parser.add_argument("--batch-size", type=int, default=4096) - parser.add_argument("--hidden-dim", type=int, default=128) - parser.add_argument("--dropout", type=float, default=0.0) - parser.add_argument("--lr", type=float, default=1e-3) - parser.add_argument("--weight-decay", type=float, default=1e-5) - parser.add_argument("--val-fraction", type=float, default=0.1) - parser.add_argument("--grad-clip", type=float, default=1.0) - parser.add_argument("--seed", type=int, default=1234) - parser.add_argument("--log-every", type=int, default=5) - parser.add_argument( - "--device", - type=str, - default="cpu", - help="Use cpu for portable training/export; cuda is okay if available.", - ) - - return parser.parse_args() - - -if __name__ == "__main__": - train(parse_args()) diff --git a/src/surrogate/ml_models/component_level/train_torch_jit_packet_latency.py b/src/surrogate/ml_models/component_level/train_torch_jit_packet_latency.py new file mode 100755 index 00000000..301ca71f --- /dev/null +++ b/src/surrogate/ml_models/component_level/train_torch_jit_packet_latency.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Train/export a TorchScript terminal/default packet-latency model. + +Input CSV rows come from PARAMS.save_packet_latency_path. The model contract is: + input FloatTensor[*,12] + output FloatTensor[*,2] = [travel_delta, next_packet_delay] +""" +import argparse +from pathlib import Path +import glob + +import numpy as np +import pandas as pd +import torch + +def resolve_device(name: str) -> torch.device: + if name == "auto": + name = "cuda" if torch.cuda.is_available() else "cpu" + if name == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was requested, but torch.cuda.is_available() is false.") + device = torch.device(name) + print(f"Using device: {device}") + if device.type == "cuda": + print(f"CUDA device: {torch.cuda.get_device_name(0)}") + return device + +from torch import nn +from torch.utils.data import DataLoader, TensorDataset + +FEATURE_COLUMNS = [ + "src_terminal", "dest_terminal", "packet_size", "is_there_another_pckt_in_queue", + "caller_lp_gid", "src_router_id", "src_group_id", "dst_router_id", "dst_group_id", + "terminal_queue_length", "terminal_vc_occupancy", "processing_packet_delay", +] +TARGET_COLUMNS = ["travel_end_time_delta", "next_packet_delay"] +ALL_COLUMNS = FEATURE_COLUMNS + TARGET_COLUMNS + [ + "packet_id", "is_surrogate_on", "is_predicted", "workload_injection", "start", "end", "latency" +] + +class MLP(nn.Module): + def __init__(self, in_dim: int, out_dim: int, hidden: int): + super().__init__() + self.net = nn.Sequential( + nn.Linear(in_dim, hidden), nn.ReLU(), + nn.Linear(hidden, hidden), nn.ReLU(), + nn.Linear(hidden, out_dim), + ) + def forward(self, x): + return self.net(x.float()) + +def read_trace(path: str) -> pd.DataFrame: + p = Path(path) + files = sorted(glob.glob(str(p / "packets-delay-gid=*.txt"))) if p.is_dir() else [str(p)] + if not files: + raise SystemExit(f"No packet latency trace files found under {path}") + frames = [pd.read_csv(f, comment="#", names=ALL_COLUMNS) for f in files] + df = pd.concat(frames, ignore_index=True) + df = df[(df["is_surrogate_on"] == 0) & (df["is_predicted"] == 0)] + df = df.replace([np.inf, -np.inf], np.nan).dropna(subset=FEATURE_COLUMNS + TARGET_COLUMNS) + df = df[(df["travel_end_time_delta"] > 0.0) & (df["next_packet_delay"] >= 0.0)] + if df.empty: + raise SystemExit("No usable high-fidelity packet latency rows after filtering") + return df + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--trace", required=True, help="Trace file or directory from save_packet_latency_path") + ap.add_argument("--out", required=True, help="Output TorchScript .pt path") + ap.add_argument("--epochs", type=int, default=30) + ap.add_argument("--batch-size", type=int, default=1024) + ap.add_argument("--hidden", type=int, default=128) + ap.add_argument("--lr", type=float, default=1e-3) + ap.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda"], + help="Training device. auto uses CUDA if available, otherwise CPU.") + args = ap.parse_args() + device = resolve_device(args.device) + + df = read_trace(args.trace) + x = torch.tensor(df[FEATURE_COLUMNS].to_numpy(dtype=np.float32), device=device) + y = torch.tensor(df[TARGET_COLUMNS].to_numpy(dtype=np.float32), device=device) + + x_mu, x_sigma = x.mean(0), x.std(0).clamp_min(1e-6) + y_mu, y_sigma = y.mean(0), y.std(0).clamp_min(1e-6) + x_n = (x - x_mu) / x_sigma + y_n = (y - y_mu) / y_sigma + + model = MLP(len(FEATURE_COLUMNS), len(TARGET_COLUMNS), args.hidden).to(device) + opt = torch.optim.AdamW(model.parameters(), lr=args.lr) + loss_fn = nn.SmoothL1Loss() + loader = DataLoader(TensorDataset(x_n, y_n), batch_size=args.batch_size, shuffle=True) + + model.train() + for epoch in range(args.epochs): + losses = [] + for xb, yb in loader: + opt.zero_grad(set_to_none=True) + loss = loss_fn(model(xb), yb) + loss.backward() + opt.step() + losses.append(float(loss.detach())) + print(f"epoch={epoch+1} loss={np.mean(losses):.6g}") + + class NormalizedWrapper(nn.Module): + def __init__(self, base, x_mu, x_sigma, y_mu, y_sigma): + super().__init__() + self.base = base.eval() + self.register_buffer("x_mu", x_mu) + self.register_buffer("x_sigma", x_sigma) + self.register_buffer("y_mu", y_mu) + self.register_buffer("y_sigma", y_sigma) + def forward(self, x): + y = self.base((x.float() - self.x_mu) / self.x_sigma) + return y * self.y_sigma + self.y_mu + + # Export a CPU TorchScript module. Training may happen on CUDA, but the + # C++ surrogate path should be able to load and run this model on CPU. + model = model.to("cpu").eval() + x_mu = x_mu.detach().to("cpu") + x_sigma = x_sigma.detach().to("cpu") + y_mu = y_mu.detach().to("cpu") + y_sigma = y_sigma.detach().to("cpu") + + wrapped = NormalizedWrapper(model, x_mu, x_sigma, y_mu, y_sigma).to("cpu").eval() + example = torch.zeros(1, len(FEATURE_COLUMNS), dtype=torch.float32) + + with torch.no_grad(): + traced = torch.jit.trace(wrapped, example) + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + traced.save(args.out) + print(f"saved {args.out}") + +if __name__ == "__main__": + main() diff --git a/src/surrogate/ml_models/component_level/train_torch_jit_router_timing.py b/src/surrogate/ml_models/component_level/train_torch_jit_router_timing.py new file mode 100755 index 00000000..974d2188 --- /dev/null +++ b/src/surrogate/ml_models/component_level/train_torch_jit_router_timing.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Train/export a TorchScript router queueing-delay model. + +Input CSV rows come from PARAMS.save_router_timing_trace_path. The model contract is: + input FloatTensor[*,12] + output FloatTensor[*,1] = [router_queueing_delay] +""" +import argparse +from pathlib import Path +import glob + +import numpy as np +import pandas as pd +import torch + +def resolve_device(name: str) -> torch.device: + if name == "auto": + name = "cuda" if torch.cuda.is_available() else "cpu" + if name == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was requested, but torch.cuda.is_available() is false.") + device = torch.device(name) + print(f"Using device: {device}") + if device.type == "cuda": + print(f"CUDA device: {torch.cuda.get_device_name(0)}") + return device + +from torch import nn +from torch.utils.data import DataLoader, TensorDataset + +FEATURE_COLUMNS = [ + "router_id", "group_id", "output_port", "output_chan", "to_terminal", "is_global", + "packet_size", "chunk_size", "output_vc_occupancy", "output_queued_count", + "next_output_available_delta", "nominal_router_delay", +] +TARGET_COLUMNS = ["router_queueing_delay"] +ALL_COLUMNS = FEATURE_COLUMNS + TARGET_COLUMNS + [ + "actual_router_delay", "is_surrogate_on", "is_predicted", "now", "packet_id", + "src_terminal", "dest_terminal", "next_stop" +] + +class MLP(nn.Module): + def __init__(self, in_dim: int, out_dim: int, hidden: int): + super().__init__() + self.net = nn.Sequential( + nn.Linear(in_dim, hidden), nn.ReLU(), + nn.Linear(hidden, hidden), nn.ReLU(), + nn.Linear(hidden, out_dim), + ) + def forward(self, x): + return self.net(x.float()) + +def _filter_router_rows(df: pd.DataFrame) -> pd.DataFrame: + df = df[(df["is_surrogate_on"] == 0) & (df["is_predicted"] == 0)] + df = df.replace([np.inf, -np.inf], np.nan).dropna(subset=FEATURE_COLUMNS + TARGET_COLUMNS) + df = df[df["router_queueing_delay"] >= 0.0] + return df + + +def read_trace(path: str, max_rows: int, seed: int, chunksize: int) -> pd.DataFrame: + """Read router timing traces without materializing the full trace. + + If max_rows > 0, maintain a bounded random sample while streaming chunks. + This still trains on real high-fidelity rows, but avoids loading a 40+ GB + CSV trace into memory at once. + """ + p = Path(path) + files = sorted(glob.glob(str(p / "router-timing-gid=*.txt"))) if p.is_dir() else [str(p)] + if not files: + raise SystemExit(f"No router timing trace files found under {path}") + + rng = np.random.default_rng(seed) + kept = [] + total_seen = 0 + total_usable = 0 + + for f in files: + print(f"reading {f}", flush=True) + reader = pd.read_csv( + f, + comment="#", + names=ALL_COLUMNS, + chunksize=chunksize, + ) + + for chunk in reader: + total_seen += len(chunk) + chunk = _filter_router_rows(chunk) + if chunk.empty: + continue + + total_usable += len(chunk) + + if max_rows <= 0: + kept.append(chunk) + continue + + kept.append(chunk) + df = pd.concat(kept, ignore_index=True) + + if len(df) > max_rows: + # Randomly keep max_rows rows from all usable rows seen so far. + # Use a fresh integer seed from rng so repeated chunks do not + # produce identical samples. + sample_seed = int(rng.integers(0, 2**31 - 1)) + df = df.sample(n=max_rows, random_state=sample_seed).reset_index(drop=True) + + kept = [df] + + if not kept: + raise SystemExit("No usable high-fidelity router timing rows after filtering") + + df = pd.concat(kept, ignore_index=True) + + if max_rows > 0 and len(df) > max_rows: + df = df.sample(n=max_rows, random_state=seed).reset_index(drop=True) + + print(f"router trace rows seen: {total_seen}", flush=True) + print(f"usable high-fidelity rows seen: {total_usable}", flush=True) + print(f"training rows kept: {len(df)}", flush=True) + + if df.empty: + raise SystemExit("No usable high-fidelity router timing rows after filtering") + + return df + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--trace", required=True, help="Trace file or directory from save_router_timing_trace_path") + ap.add_argument("--out", required=True, help="Output TorchScript .pt path") + ap.add_argument("--epochs", type=int, default=30) + ap.add_argument("--batch-size", type=int, default=1024) + ap.add_argument("--hidden", type=int, default=128) + ap.add_argument("--lr", type=float, default=1e-3) + ap.add_argument("--device", default="auto", choices=["auto", "cpu", "cuda"], + help="Training device. auto uses CUDA if available, otherwise CPU.") + ap.add_argument("--max-rows", type=int, default=2000000, + help="Maximum usable router rows to keep. Use <=0 to load all rows.") + ap.add_argument("--read-chunksize", type=int, default=1000000, + help="CSV rows per pandas chunk while streaming router traces.") + ap.add_argument("--seed", type=int, default=0) + args = ap.parse_args() + device = resolve_device(args.device) + + df = read_trace( + args.trace, + max_rows=args.max_rows, + seed=args.seed, + chunksize=args.read_chunksize, + ) + x = torch.tensor(df[FEATURE_COLUMNS].to_numpy(dtype=np.float32), device=device) + y = torch.tensor(df[TARGET_COLUMNS].to_numpy(dtype=np.float32), device=device) + + x_mu, x_sigma = x.mean(0), x.std(0).clamp_min(1e-6) + y_mu, y_sigma = y.mean(0), y.std(0).clamp_min(1e-6) + x_n = (x - x_mu) / x_sigma + y_n = (y - y_mu) / y_sigma + + model = MLP(len(FEATURE_COLUMNS), len(TARGET_COLUMNS), args.hidden).to(device) + opt = torch.optim.AdamW(model.parameters(), lr=args.lr) + loss_fn = nn.SmoothL1Loss() + loader = DataLoader(TensorDataset(x_n, y_n), batch_size=args.batch_size, shuffle=True) + + model.train() + for epoch in range(args.epochs): + losses = [] + for xb, yb in loader: + opt.zero_grad(set_to_none=True) + loss = loss_fn(model(xb), yb) + loss.backward() + opt.step() + losses.append(float(loss.detach())) + print(f"epoch={epoch+1} loss={np.mean(losses):.6g}") + + class NormalizedWrapper(nn.Module): + def __init__(self, base, x_mu, x_sigma, y_mu, y_sigma): + super().__init__() + self.base = base.eval() + self.register_buffer("x_mu", x_mu) + self.register_buffer("x_sigma", x_sigma) + self.register_buffer("y_mu", y_mu) + self.register_buffer("y_sigma", y_sigma) + def forward(self, x): + y = self.base((x.float() - self.x_mu) / self.x_sigma) + return y * self.y_sigma + self.y_mu + + # Export a CPU TorchScript module. Training may happen on CUDA, but the + # C++ surrogate path should be able to load and run this model on CPU. + model = model.to("cpu").eval() + x_mu = x_mu.detach().to("cpu") + x_sigma = x_sigma.detach().to("cpu") + y_mu = y_mu.detach().to("cpu") + y_sigma = y_sigma.detach().to("cpu") + + wrapped = NormalizedWrapper(model, x_mu, x_sigma, y_mu, y_sigma).to("cpu").eval() + example = torch.zeros(1, len(FEATURE_COLUMNS), dtype=torch.float32) + + with torch.no_grad(): + traced = torch.jit.trace(wrapped, example) + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + traced.save(args.out) + print(f"saved {args.out}") + +if __name__ == "__main__": + main() diff --git a/src/surrogate/packet-latency-predictor/average.c b/src/surrogate/packet-latency-predictor/average.c index 59b9e7d2..2beeb5e7 100644 --- a/src/surrogate/packet-latency-predictor/average.c +++ b/src/surrogate/packet-latency-predictor/average.c @@ -3,6 +3,11 @@ double ignore_until = 0; static int num_terminals = 0; +static int average_predictor_debug_prints = 0; + +void average_latency_predictor_set_debug_prints(int enabled) { + average_predictor_debug_prints = enabled ? 1 : 0; +} // === Average packet latency functionality @@ -90,7 +95,8 @@ static struct packet_end predict_latency(struct latency_surrogate * data, tw_lp static unsigned long long average_predict_calls = 0; average_predict_calls++; - if (average_predict_calls <= 20 || average_predict_calls % 10000 == 0) { + if (average_predictor_debug_prints && + (average_predict_calls <= 20 || average_predict_calls % 10000 == 0)) { fprintf(stderr, "[average predict debug] calls=%llu latency=%f next_packet_delay=%f " "next_delay_points=%u total_latency_points=%u dest_latency_points=%u\n", diff --git a/src/surrogate/packet-latency-predictor/torch-jit.C b/src/surrogate/packet-latency-predictor/torch-jit.C index 2842a705..acfbb187 100644 --- a/src/surrogate/packet-latency-predictor/torch-jit.C +++ b/src/surrogate/packet-latency-predictor/torch-jit.C @@ -12,19 +12,35 @@ #include #include +/* Backward-compatible global model used by the two original Torch-JIT modes. */ static torch::jit::Module packet_latency_model; +/* New optional models used only by torch_jit_mode="lp-aware-lp-type-models". */ +static torch::jit::Module terminal_packet_latency_model; +static torch::jit::Module default_packet_latency_model; +static torch::jit::Module router_timing_model; + +static bool torch_jit_lp_aware_mode = false; +static bool torch_jit_lp_type_model_mode = false; +static bool terminal_packet_latency_model_loaded = false; +static bool default_packet_latency_model_loaded = false; +static bool router_timing_model_loaded = false; +static bool torch_jit_debug_prints = false; + static uint64_t torch_jit_predict_calls = 0; +static uint64_t torch_jit_router_predict_calls = 0; static double torch_jit_forward_total_sec = 0.0; +static double torch_jit_router_forward_total_sec = 0.0; static double torch_jit_min_travel_delta = 1.0e300; static double torch_jit_max_travel_delta = -1.0e300; static double torch_jit_min_next_delay = 1.0e300; static double torch_jit_max_next_delay = -1.0e300; -static bool torch_jit_lp_aware_mode = false; -static bool torch_jit_debug_prints = false; +static double torch_jit_min_router_queueing_delay = 1.0e300; +static double torch_jit_max_router_queueing_delay = -1.0e300; static constexpr int TORCH_JIT_LEGACY_FEATURE_COUNT = 4; static constexpr int TORCH_JIT_LP_AWARE_FEATURE_COUNT = 12; +static constexpr int TORCH_JIT_ROUTER_TIMING_FEATURE_COUNT = 12; void surrogate_torch_set_lp_aware_mode(bool enabled) { torch_jit_lp_aware_mode = enabled; @@ -34,6 +50,74 @@ void surrogate_torch_set_debug_prints(bool enabled) { torch_jit_debug_prints = enabled; } +static bool has_path(char const *path) { + return path != nullptr && path[0] != '\0'; +} + +static void load_module_or_die(torch::jit::Module *module, char const *path, char const *label) { + if (!has_path(path)) { + tw_error(TW_LOC, "Missing Torch-JIT %s path", label); + } + + try { + *module = torch::jit::load(path); + } catch (const c10::Error& e) { + tw_error(TW_LOC, "Error loading Torch-JIT %s model from `%s`", label, path); + } + + if (module->is_training()) { + std::cerr << "The Torch-JIT " << label + << " model was saved before running .eval(); inference will use training-mode behavior." + << std::endl; + } +} + +static void validate_output_dims(at::Tensor const &output, int expected_last_dim, char const *label) { + int const dims = output.ndimension(); + if (dims < 1) { + tw_error(TW_LOC, "Torch-JIT %s model returned a scalar; expected [1,%d]", label, expected_last_dim); + } + for (int i = 0; i < dims - 1; i++) { + if (at::size(output, i) != 1) { + tw_error(TW_LOC, "Torch-JIT %s model returned unexpected dim %d size %lld; expected 1", + label, i, (long long)at::size(output, i)); + } + } + if (at::size(output, dims - 1) != expected_last_dim) { + tw_error(TW_LOC, "Torch-JIT %s model returned last dim %lld; expected %d", + label, (long long)at::size(output, dims - 1), expected_last_dim); + } +} + +static void validate_packet_latency_model(torch::jit::Module *model, int feature_count, at::ScalarType dtype, char const *label) { + std::vector inputs; + torch::NoGradGuard no_grad; + + if (dtype == at::kFloat) { + std::vector data_input(feature_count, 0.0f); + inputs.emplace_back(torch::from_blob(data_input.data(), {1, feature_count}, at::kFloat).clone()); + } else if (dtype == at::kLong) { + std::vector data_input(feature_count, 0); + inputs.emplace_back(torch::from_blob(data_input.data(), {1, feature_count}, at::kLong).clone()); + } else { + tw_error(TW_LOC, "Unsupported Torch-JIT validation dtype for %s", label); + } + + at::Tensor output = model->forward(inputs).toTensor(); + validate_output_dims(output, 2, label); +} + +static void validate_router_timing_model(torch::jit::Module *model, char const *label) { + std::vector inputs; + torch::NoGradGuard no_grad; + std::vector data_input(TORCH_JIT_ROUTER_TIMING_FEATURE_COUNT, 0.0f); + inputs.emplace_back(torch::from_blob(data_input.data(), + {1, TORCH_JIT_ROUTER_TIMING_FEATURE_COUNT}, + at::kFloat).clone()); + at::Tensor output = model->forward(inputs).toTensor(); + validate_output_dims(output, 1, label); +} + static std::vector build_lp_aware_packet_features( tw_lp *lp, unsigned int src_terminal, @@ -42,13 +126,10 @@ static std::vector build_lp_aware_packet_features( assert(packet_dest != nullptr); return { - /* Existing four features first. */ (float)src_terminal, (float)packet_dest->dfdally_dest_terminal_id, (float)packet_dest->packet_size, packet_dest->is_there_another_pckt_in_queue ? 1.0f : 0.0f, - - /* Explicit LP/topology/context features. */ lp ? (float)lp->gid : -1.0f, (float)packet_dest->src_router_id, (float)packet_dest->src_group_id, @@ -60,74 +141,95 @@ static std::vector build_lp_aware_packet_features( }; } +void surrogate_torch_init(char const *dir) { + std::cout << "Loading Torch-JIT packet-latency model\n"; -inline void assert_correct_dims(at::Tensor * t) { - int const dims = t->ndimension(); + torch_jit_lp_type_model_mode = false; + terminal_packet_latency_model_loaded = false; + default_packet_latency_model_loaded = false; + router_timing_model_loaded = false; - for (int i = 0; i < dims-1; i++) { - assert(at::size(*t, i) == 1); - } - assert(at::size(*t, dims - 1) == 2); -} + load_module_or_die(&packet_latency_model, dir, "packet-latency"); + at::set_num_threads(1); -void surrogate_torch_init(char const * dir) { - std::cout << "Loading Torch-JIT model\n"; - try { - // Deserialize the ScriptModule from a file - packet_latency_model = torch::jit::load(dir); - } - catch (const c10::Error& e) { - tw_error(TW_LOC, "Error loading Torch-JIT model"); + if (torch_jit_lp_aware_mode) { + validate_packet_latency_model(&packet_latency_model, + TORCH_JIT_LP_AWARE_FEATURE_COUNT, + at::kFloat, + "lp-aware packet-latency"); + } else { + validate_packet_latency_model(&packet_latency_model, + TORCH_JIT_LEGACY_FEATURE_COUNT, + at::kLong, + "legacy packet-latency"); } - // Configuring to run on a single thread + std::cout << "Torch-JIT packet-latency model loaded successfully\n"; +} + +void surrogate_torch_init_lp_type_models( + char const *terminal_model_path, + char const *router_timing_model_path, + char const *default_model_path) +{ + std::cout << "Loading Torch-JIT LP-type-aware models\n"; + + torch_jit_lp_aware_mode = true; + torch_jit_lp_type_model_mode = true; + terminal_packet_latency_model_loaded = false; + default_packet_latency_model_loaded = false; + router_timing_model_loaded = false; + at::set_num_threads(1); - // === Checking consistency of model with dummy input - if (packet_latency_model.is_training()) { - std::cerr << "The Torch-JIT model was saved before running .eval(). " - "The output from the model will be as if it was in training mode, " - "meaning, it might be faulty." - << std::endl; + if (has_path(terminal_model_path)) { + load_module_or_die(&terminal_packet_latency_model, terminal_model_path, "terminal packet-latency"); + validate_packet_latency_model(&terminal_packet_latency_model, + TORCH_JIT_LP_AWARE_FEATURE_COUNT, + at::kFloat, + "terminal packet-latency"); + terminal_packet_latency_model_loaded = true; } - std::vector inputs; - torch::NoGradGuard no_grad; + if (has_path(default_model_path)) { + load_module_or_die(&default_packet_latency_model, default_model_path, "default packet-latency"); + validate_packet_latency_model(&default_packet_latency_model, + TORCH_JIT_LP_AWARE_FEATURE_COUNT, + at::kFloat, + "default packet-latency"); + default_packet_latency_model_loaded = true; + } - if (torch_jit_lp_aware_mode) { - std::vector data_input(TORCH_JIT_LP_AWARE_FEATURE_COUNT, 0.0f); - inputs.emplace_back( - torch::from_blob( - data_input.data(), - {1, TORCH_JIT_LP_AWARE_FEATURE_COUNT}, - at::kFloat).clone()); + if (!terminal_packet_latency_model_loaded && !default_packet_latency_model_loaded) { + tw_error(TW_LOC, "torch_jit_mode=lp-aware-lp-type-models requires torch_jit_terminal_model_path or torch_jit_default_model_path"); + } + + if (has_path(router_timing_model_path)) { + load_module_or_die(&router_timing_model, router_timing_model_path, "router timing"); + validate_router_timing_model(&router_timing_model, "router timing"); + router_timing_model_loaded = true; } else { - long int data_input[] = {0, 0, 0, 0}; - inputs.emplace_back( - torch::from_blob( - data_input, - {1, TORCH_JIT_LEGACY_FEATURE_COUNT}, - at::kLong).clone()); + tw_warning(TW_LOC, "No torch_jit_router_timing_model_path configured; router timing inference disabled."); } - // Predicting value - at::Tensor output = packet_latency_model.forward(inputs).toTensor(); - assert_correct_dims(&output); - // === End of check - std::cout << "Torch-JIT model loaded successfully\n"; + std::cout << "Torch-JIT LP-type-aware models loaded successfully\n"; } +bool surrogate_torch_router_timing_model_enabled(void) { + return torch_jit_lp_type_model_mode && router_timing_model_loaded; +} -static struct packet_end surrogate_torch_predict(void *, tw_lp * lp, unsigned int src_terminal, struct packet_start const * packet_dest) { - //auto t_start = std::chrono::high_resolution_clock::now(); - - // Create a vector of inputs. +static struct packet_end surrogate_torch_predict(void *, tw_lp *lp, unsigned int src_terminal, struct packet_start const *packet_dest) { std::vector inputs; + torch::jit::Module *model = &packet_latency_model; - if (torch_jit_lp_aware_mode) { - std::vector data_input = - build_lp_aware_packet_features(lp, src_terminal, packet_dest); + if (torch_jit_lp_type_model_mode) { + model = terminal_packet_latency_model_loaded ? &terminal_packet_latency_model : &default_packet_latency_model; + } + + if (torch_jit_lp_aware_mode || torch_jit_lp_type_model_mode) { + std::vector data_input = build_lp_aware_packet_features(lp, src_terminal, packet_dest); if (torch_jit_debug_prints && torch_jit_predict_calls < 20) { fprintf(stderr, @@ -136,53 +238,40 @@ static struct packet_end surrogate_torch_predict(void *, tw_lp * lp, unsigned in "caller_lp_gid=%g src_router_id=%g src_group_id=%g " "dst_router_id=%g dst_group_id=%g terminal_queue_length=%g " "terminal_vc_occupancy=%g processing_packet_delay=%g\n", - (double)data_input[0], - (double)data_input[1], - (double)data_input[2], - (double)data_input[3], - (double)data_input[4], - (double)data_input[5], - (double)data_input[6], - (double)data_input[7], - (double)data_input[8], - (double)data_input[9], - (double)data_input[10], - (double)data_input[11]); + (double)data_input[0], (double)data_input[1], (double)data_input[2], + (double)data_input[3], (double)data_input[4], (double)data_input[5], + (double)data_input[6], (double)data_input[7], (double)data_input[8], + (double)data_input[9], (double)data_input[10], (double)data_input[11]); fflush(stderr); } assert((int)data_input.size() == TORCH_JIT_LP_AWARE_FEATURE_COUNT); - - inputs.emplace_back( - torch::from_blob( - data_input.data(), - {1, TORCH_JIT_LP_AWARE_FEATURE_COUNT}, - at::kFloat).clone()); + inputs.emplace_back(torch::from_blob(data_input.data(), + {1, TORCH_JIT_LP_AWARE_FEATURE_COUNT}, + at::kFloat).clone()); } else { long int data_input[] = { - src_terminal, - packet_dest->dfdally_dest_terminal_id, - packet_dest->packet_size, - packet_dest->is_there_another_pckt_in_queue + (long int)src_terminal, + (long int)packet_dest->dfdally_dest_terminal_id, + (long int)packet_dest->packet_size, + packet_dest->is_there_another_pckt_in_queue ? 1L : 0L }; - - inputs.emplace_back( - torch::from_blob( - data_input, - {1, TORCH_JIT_LEGACY_FEATURE_COUNT}, - at::kLong).clone()); + inputs.emplace_back(torch::from_blob(data_input, + {1, TORCH_JIT_LEGACY_FEATURE_COUNT}, + at::kLong).clone()); } + torch::NoGradGuard no_grad; auto const torch_jit_t0 = std::chrono::high_resolution_clock::now(); - at::Tensor output = packet_latency_model.forward(inputs).toTensor(); + at::Tensor output = model->forward(inputs).toTensor(); auto const torch_jit_t1 = std::chrono::high_resolution_clock::now(); - torch_jit_forward_total_sec += std::chrono::duration( - torch_jit_t1 - torch_jit_t0).count(); - //assert_correct_dims(&output); + validate_output_dims(output, 2, "packet-latency inference"); - auto *out_data = output.data_ptr(); - + torch_jit_forward_total_sec += std::chrono::duration(torch_jit_t1 - torch_jit_t0).count(); torch_jit_predict_calls++; + + output = output.to(at::kFloat).contiguous(); + auto *out_data = output.data_ptr(); double const raw_travel_delta = (double)out_data[0]; double const raw_next_delay = (double)out_data[1]; @@ -199,72 +288,118 @@ static struct packet_end surrogate_torch_predict(void *, tw_lp * lp, unsigned in ? raw_next_delay : min_next_packet_delay; - if (raw_travel_delta < torch_jit_min_travel_delta) { - torch_jit_min_travel_delta = raw_travel_delta; - } - if (raw_travel_delta > torch_jit_max_travel_delta) { - torch_jit_max_travel_delta = raw_travel_delta; - } - if (raw_next_delay < torch_jit_min_next_delay) { - torch_jit_min_next_delay = raw_next_delay; - } - if (raw_next_delay > torch_jit_max_next_delay) { - torch_jit_max_next_delay = raw_next_delay; - } + if (raw_travel_delta < torch_jit_min_travel_delta) torch_jit_min_travel_delta = raw_travel_delta; + if (raw_travel_delta > torch_jit_max_travel_delta) torch_jit_max_travel_delta = raw_travel_delta; + if (raw_next_delay < torch_jit_min_next_delay) torch_jit_min_next_delay = raw_next_delay; + if (raw_next_delay > torch_jit_max_next_delay) torch_jit_max_next_delay = raw_next_delay; if (torch_jit_debug_prints && - (torch_jit_predict_calls <= 20 || - torch_jit_predict_calls % 10000 == 0)) { + (torch_jit_predict_calls <= 20 || torch_jit_predict_calls % 10000 == 0)) { fprintf(stderr, "[torch-jit predict debug] calls=%llu avg_forward_us=%g " "raw_travel_delta=%g raw_next_delay=%g " "effective_travel_delta=%g effective_next_delay=%g " "minmax_travel=[%g,%g] minmax_next=[%g,%g]\n", (unsigned long long)torch_jit_predict_calls, - 1.0e6 * torch_jit_forward_total_sec / - (double)torch_jit_predict_calls, - raw_travel_delta, - raw_next_delay, - predicted_travel_delta, - predicted_next_packet_delay, - torch_jit_min_travel_delta, - torch_jit_max_travel_delta, - torch_jit_min_next_delay, - torch_jit_max_next_delay); + 1.0e6 * torch_jit_forward_total_sec / (double)torch_jit_predict_calls, + raw_travel_delta, raw_next_delay, + predicted_travel_delta, predicted_next_packet_delay, + torch_jit_min_travel_delta, torch_jit_max_travel_delta, + torch_jit_min_next_delay, torch_jit_max_next_delay); fflush(stderr); } -return (struct packet_end) { + + return (struct packet_end) { .travel_end_time = packet_dest->travel_start_time + predicted_travel_delta, .next_packet_delay = predicted_next_packet_delay, }; - - //auto t_end = std::chrono::high_resolution_clock::now(); - //double total = std::chrono::duration(t_end-t_start).count(); } +double surrogate_torch_predict_router_queueing_delay( + struct router_timing_prediction_start const *start, + double fallback_queueing_delay) +{ + if (!surrogate_torch_router_timing_model_enabled() || start == nullptr) { + return fallback_queueing_delay; + } -// Dummies to use when no actual data is fed -static void init_pred_dummy(void * data, tw_lp * lp, unsigned int src_terminal) { - (void) data; - (void) lp; - (void) src_terminal; -} + std::vector data_input = { + start->router_id, + start->group_id, + start->output_port, + start->output_chan, + start->to_terminal, + start->is_global, + start->packet_size, + start->chunk_size, + start->output_vc_occupancy, + start->output_queued_count, + start->next_output_available_delta, + start->nominal_router_delay + }; + assert((int)data_input.size() == TORCH_JIT_ROUTER_TIMING_FEATURE_COUNT); + + std::vector inputs; + inputs.emplace_back(torch::from_blob(data_input.data(), + {1, TORCH_JIT_ROUTER_TIMING_FEATURE_COUNT}, + at::kFloat).clone()); + + torch::NoGradGuard no_grad; + auto const torch_jit_t0 = std::chrono::high_resolution_clock::now(); + at::Tensor output = router_timing_model.forward(inputs).toTensor(); + auto const torch_jit_t1 = std::chrono::high_resolution_clock::now(); + validate_output_dims(output, 1, "router timing inference"); + + torch_jit_router_forward_total_sec += std::chrono::duration(torch_jit_t1 - torch_jit_t0).count(); + torch_jit_router_predict_calls++; + + output = output.to(at::kFloat).contiguous(); + double const raw_queueing_delay = (double)output.data_ptr()[0]; + double const predicted_queueing_delay = + std::isfinite(raw_queueing_delay) && raw_queueing_delay >= 0.0 + ? raw_queueing_delay + : fallback_queueing_delay; + if (raw_queueing_delay < torch_jit_min_router_queueing_delay) torch_jit_min_router_queueing_delay = raw_queueing_delay; + if (raw_queueing_delay > torch_jit_max_router_queueing_delay) torch_jit_max_router_queueing_delay = raw_queueing_delay; -static void feed_pred_dummy(struct latency_surrogate * data, tw_lp * lp, unsigned int src_terminal, struct packet_start const * start, struct packet_end const * end) { - (void) data; - (void) lp; - (void) src_terminal; - (void) start; - (void) end; + if (torch_jit_debug_prints && + (torch_jit_router_predict_calls <= 20 || torch_jit_router_predict_calls % 10000 == 0)) { + fprintf(stderr, + "[torch-jit router timing debug] calls=%llu avg_forward_us=%g " + "raw_queueing_delay=%g effective_queueing_delay=%g minmax_queueing=[%g,%g]\n", + (unsigned long long)torch_jit_router_predict_calls, + 1.0e6 * torch_jit_router_forward_total_sec / (double)torch_jit_router_predict_calls, + raw_queueing_delay, + predicted_queueing_delay, + torch_jit_min_router_queueing_delay, + torch_jit_max_router_queueing_delay); + fflush(stderr); + } + + return predicted_queueing_delay; } +// Dummies to use when no actual data is fed +static void init_pred_dummy(void *data, tw_lp *lp, unsigned int src_terminal) { + (void)data; + (void)lp; + (void)src_terminal; +} -static void predict_latency_rc_dummy(struct latency_surrogate * data, tw_lp * lp) { - (void) data; - (void) lp; +static void feed_pred_dummy(struct latency_surrogate *data, tw_lp *lp, unsigned int src_terminal, + struct packet_start const *start, struct packet_end const *end) { + (void)data; + (void)lp; + (void)src_terminal; + (void)start; + (void)end; } +static void predict_latency_rc_dummy(struct latency_surrogate *data, tw_lp *lp) { + (void)data; + (void)lp; +} struct packet_latency_predictor torch_latency_predictor = { .init = (init_pred_lat_f) init_pred_dummy, From 47d9d843faeb59ac596545d9f95f35ef498d9fec Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Mon, 1 Jun 2026 11:10:32 -0400 Subject: [PATCH 13/13] Make zeromq dependency handling robust --- CODES-compile-instructions.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CODES-compile-instructions.sh b/CODES-compile-instructions.sh index 1ba6f3c6..9eb00143 100644 --- a/CODES-compile-instructions.sh +++ b/CODES-compile-instructions.sh @@ -108,6 +108,32 @@ if [ $union_enable = 1 ]; then fi + +# Make system pkg-config metadata visible even when Conda's pkg-config is active. +# This is needed for libzmq.pc on systems where ZeroMQ is installed through the OS +# but the active Conda environment's pkg-config only searches Conda pkgconfig dirs. +if ! pkg-config --exists libzmq 2>/dev/null; then + for pcdir in \ + /usr/lib/x86_64-linux-gnu/pkgconfig \ + /usr/lib64/pkgconfig \ + /usr/lib/pkgconfig \ + /usr/local/lib/pkgconfig \ + /usr/local/lib64/pkgconfig \ + /opt/homebrew/lib/pkgconfig \ + /usr/share/pkgconfig + do + if [ -d "$pcdir" ]; then + export PKG_CONFIG_PATH="$pcdir:${PKG_CONFIG_PATH:-}" + fi + done +fi + +if ! pkg-config --exists libzmq 2>/dev/null; then + echo "WARNING: pkg-config still cannot find libzmq.pc." >&2 + echo " If ZMQML fails to build, install the ZeroMQ development package" >&2 + echo " or set PKG_CONFIG_PATH to the directory containing libzmq.pc." >&2 +fi + # Build local ZMQML requester library required by director-client.C pushd codes/src/surrogate/zmqml make clean