Add ability to bypass profiling entirely by making ProfilingSettings optional

bin/run-model/src/run-model/main.cc

@@ -101,7 +101,7 @@ int main(int argc, char **argv) {
             /*optimizer=*/optimizer_attrs,
             /*loss=*/std::nullopt,
             /*input_tensors=*/input_tensors,
-            /*profiling_settings=*/ProfilingSettings{0_n, 1_p},
+            /*profiling_settings=*/std::nullopt,
             /*device_handle=*/device_handle,
             /*device_type=*/DeviceType::GPU);
 
@@ -110,7 +110,7 @@ int main(int argc, char **argv) {
         for (int i = 0; i < num_epochs; i++) {
           perform_all_passes_for_pcg_instance(
               /*instance=*/pcg_instance,
-              /*profiling_settings=*/ProfilingSettings{0_n, 1_p},
+              /*profiling_settings=*/std::nullopt,
               /*device_handle=*/device_handle);
         }
       });

lib/local-execution/include/local-execution/computation_graph_instance.h

@@ -46,31 +46,30 @@ ComputationGraphInstance create_computation_graph_instance(
     std::optional<LossConfig> const &loss,
     std::map<DynamicValueAttrs, DynamicTensorAccessor> const &input_tensors,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
     device_handle_t const &device_handle,
     global_device_id_t global_device_id);
 
 std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
     perform_all_passes_for_computation_graph_instance(
         ComputationGraphInstance &instance,
-        ProfilingSettings const &profiling_settings,
+        std::optional<ProfilingSettings> const &profiling_settings,
         device_handle_t const &ff_handle,
         global_device_id_t global_device_id);
 std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
     perform_forward_pass_for_computation_graph_instance(
         ComputationGraphInstance const &instance,
-        ProfilingSettings const &profiling_settings,
+        std::optional<ProfilingSettings> const &profiling_settings,
         device_handle_t const &ff_handle,
         global_device_id_t global_device_id);
 std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
     perform_backward_pass_for_computation_graph_instance(
         ComputationGraphInstance const &instance,
-        ProfilingSettings const &profiling_settings,
+        std::optional<ProfilingSettings> const &profiling_settings,
         device_handle_t const &ff_handle,
         global_device_id_t global_device_id);
 void perform_update_pass_for_computation_graph_instance(
     ComputationGraphInstance &instance,
-    ProfilingSettings const &profiling_settings,
+    std::optional<ProfilingSettings> const &profiling_settings,
     device_handle_t const &ff_handle,
     global_device_id_t global_device_id);
 

lib/local-execution/include/local-execution/local_task_argument_accessor.h

@@ -15,7 +15,7 @@ struct LocalTaskArgumentAccessor : public ITaskArgumentAccessor {
       Allocator const &allocator,
       std::map<TaskTensorParameter, DynamicTensorAccessor> const
           &tensor_slots_backing,
-      ProfilingSettings const &profiling_settings,
+      std::optional<ProfilingSettings> const &profiling_settings,
       device_handle_t const &ff_handle,
       std::optional<PCGOperatorAttrs> const &op_attrs,
       std::optional<LossAttrs> const &loss_attrs,
@@ -31,7 +31,7 @@ struct LocalTaskArgumentAccessor : public ITaskArgumentAccessor {
   GenericTensorAccessor get_tensor(TaskTensorParameter slot,
                                    Permissions priv) const override;
 
-  ProfilingSettings get_profiling_settings() const override;
+  std::optional<ProfilingSettings> get_profiling_settings() const override;
   device_handle_t get_ff_handle() const override;
   DeviceType get_kernel_device_type() const override;
   PCGOperatorAttrs get_op_attrs() const override;
@@ -47,7 +47,7 @@ struct LocalTaskArgumentAccessor : public ITaskArgumentAccessor {
   Allocator allocator;
   std::map<TaskTensorParameter, DynamicTensorAccessor> tensor_slots_backing;
 
-  ProfilingSettings profiling_settings;
+  std::optional<ProfilingSettings> profiling_settings;
   device_handle_t ff_handle;
   DeviceType kernel_device_type;
   std::optional<PCGOperatorAttrs> op_attrs;

lib/local-execution/include/local-execution/per_device_op_state_initialization.h

@@ -12,21 +12,18 @@ namespace FlexFlow {
 
 bool no_nodes_are_initialized(DynamicOpenDataflowGraph const &g);
 
-DynamicNodeInvocation
-    initialize_node(DynamicNodeInvocation const &i,
-                    Allocator &allocator,
-                    ProfilingSettings const &profiling_settings,
-                    device_handle_t const &device_handle,
-                    OptimizerAttrs const &optimizer_attrs,
-                    global_device_id_t device_idx);
+DynamicNodeInvocation initialize_node(DynamicNodeInvocation const &i,
+                                      Allocator &allocator,
+                                      device_handle_t const &device_handle,
+                                      OptimizerAttrs const &optimizer_attrs,
+                                      global_device_id_t device_idx);
 
 /**
  * @brief Initialize all operators and save the per-device op state
  */
 DynamicOpenDataflowGraph perform_per_device_op_state_initialization(
     DynamicOpenDataflowGraph const &,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
     device_handle_t const &device_handle,
     OptimizerAttrs const &optimizer_attrs,
     global_device_id_t device_idx);

lib/local-execution/include/local-execution/task_execution.h

@@ -12,7 +12,7 @@ namespace FlexFlow {
 TaskArgumentAccessor make_task_argument_accessor_for_invocation(
     DynamicNodeInvocation const &invocation,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
+    std::optional<ProfilingSettings> const &profiling_settings,
     device_handle_t const &ff_handle,
     std::optional<PerDeviceOpState> const &per_device_op_state,
     std::optional<OptimizerAttrs> const &optimizer_attrs,
@@ -21,7 +21,7 @@ TaskArgumentAccessor make_task_argument_accessor_for_invocation(
 std::optional<milliseconds_t> execute_dynamic_node_invocation(
     DynamicNodeInvocation const &invocation,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
+    std::optional<ProfilingSettings> const &profiling_settings,
     device_handle_t const &ff_handle,
     std::optional<PerDeviceOpState> const &per_device_op_state,
     std::optional<OptimizerAttrs> const &optimizer_attrs,

lib/local-execution/src/local-execution/computation_graph_instance.cc

@@ -64,7 +64,6 @@ ComputationGraphInstance create_computation_graph_instance(
     std::optional<LossConfig> const &loss,
     std::map<DynamicValueAttrs, DynamicTensorAccessor> const &input_tensors,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
     device_handle_t const &device_handle,
     global_device_id_t device_idx) {
   DynamicOpenDataflowGraph dg = make_dynamic_open_dataflow_graph_from_cg(cg);
@@ -89,12 +88,8 @@ ComputationGraphInstance create_computation_graph_instance(
         return get_loss_tensor_accessor(dg, lgv);
       });
 
-  dg = perform_per_device_op_state_initialization(dg,
-                                                  allocator,
-                                                  profiling_settings,
-                                                  device_handle,
-                                                  optimizer_attrs,
-                                                  device_idx);
+  dg = perform_per_device_op_state_initialization(
+      dg, allocator, device_handle, optimizer_attrs, device_idx);
 
   // Compute the topological ordering of the graph
   auto [kwarg_graph, node_map] =
@@ -112,7 +107,7 @@ static std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
         std::vector<DynamicNodeInvocation> const &invocations,
         Allocator &allocator,
         OptimizerAttrs const &optimizer_attrs,
-        ProfilingSettings const &profiling_settings,
+        std::optional<ProfilingSettings> const &profiling_settings,
         device_handle_t const &ff_handle,
         global_device_id_t device_idx) {
   return map_from_pairs(
@@ -137,7 +132,7 @@ static std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
 std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
     perform_all_passes_for_computation_graph_instance(
         ComputationGraphInstance &instance,
-        ProfilingSettings const &profiling_settings,
+        std::optional<ProfilingSettings> const &profiling_settings,
         device_handle_t const &ff_handle,
         global_device_id_t device_idx) {
   std::vector<DynamicNodeInvocation> execution_order =
@@ -157,7 +152,7 @@ std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
 std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
     perform_forward_pass_for_computation_graph_instance(
         ComputationGraphInstance const &instance,
-        ProfilingSettings const &profiling_settings,
+        std::optional<ProfilingSettings> const &profiling_settings,
         device_handle_t const &ff_handle,
         global_device_id_t device_idx) {
   std::vector<DynamicNodeInvocation> execution_order =
@@ -180,7 +175,7 @@ std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
 std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
     perform_backward_pass_for_computation_graph_instance(
         ComputationGraphInstance const &instance,
-        ProfilingSettings const &profiling_settings,
+        std::optional<ProfilingSettings> const &profiling_settings,
         device_handle_t const &ff_handle,
         global_device_id_t device_idx) {
   std::vector<DynamicNodeInvocation> execution_order =
@@ -202,7 +197,7 @@ std::map<dynamic_layer_guid_t, std::optional<milliseconds_t>>
 
 void perform_update_pass_for_computation_graph_instance(
     ComputationGraphInstance &instance,
-    ProfilingSettings const &profiling_settings,
+    std::optional<ProfilingSettings> const &profiling_settings,
     device_handle_t const &ff_handle,
     global_device_id_t device_idx) {
   std::vector<DynamicNodeInvocation> execution_order =

lib/local-execution/src/local-execution/cost_estimator/local_cost_estimator.cc

@@ -122,7 +122,6 @@ OpCostMetrics LocalCostEstimator::estimate_cost(
       /*loss=*/std::nullopt,
       /*input_tensors=*/{},
       /*allocator=*/allocator,
-      /*profiling_settings=*/this->profiling_settings,
       /*device_handle=*/this->device_handle,
       /*device_idx=*/this->device_idx);
 

lib/local-execution/src/local-execution/local_task_argument_accessor.cc

@@ -10,7 +10,7 @@ LocalTaskArgumentAccessor::LocalTaskArgumentAccessor(
     Allocator const &allocator,
     std::map<TaskTensorParameter, DynamicTensorAccessor> const
         &tensor_slots_backing,
-    ProfilingSettings const &profiling_settings,
+    std::optional<ProfilingSettings> const &profiling_settings,
     device_handle_t const &ff_handle,
     std::optional<PCGOperatorAttrs> const &op_attrs,
     std::optional<LossAttrs> const &loss_attrs,
@@ -73,7 +73,8 @@ GenericTensorAccessor
   }
 }
 
-ProfilingSettings LocalTaskArgumentAccessor::get_profiling_settings() const {
+std::optional<ProfilingSettings>
+    LocalTaskArgumentAccessor::get_profiling_settings() const {
   return this->profiling_settings;
 }
 

lib/local-execution/src/local-execution/per_device_op_state_initialization.cc

@@ -18,13 +18,11 @@ bool no_nodes_are_initialized(DynamicOpenDataflowGraph const &g) {
       }));
 }
 
-DynamicNodeInvocation
-    initialize_node(DynamicNodeInvocation const &i,
-                    Allocator &allocator,
-                    ProfilingSettings const &profiling_settings,
-                    device_handle_t const &device_handle,
-                    OptimizerAttrs const &optimizer_attrs,
-                    global_device_id_t device_idx) {
+DynamicNodeInvocation initialize_node(DynamicNodeInvocation const &i,
+                                      Allocator &allocator,
+                                      device_handle_t const &device_handle,
+                                      OptimizerAttrs const &optimizer_attrs,
+                                      global_device_id_t device_idx) {
   if (!i.node_attrs.op_attrs.has_value() ||
       !i.node_attrs.op_attrs.value().is_pcg_op()) {
     return i;
@@ -40,7 +38,7 @@ DynamicNodeInvocation
       make_task_argument_accessor_for_invocation(
           /*invocation=*/i,
           /*allocator=*/allocator,
-          /*profiling_settings=*/profiling_settings,
+          /*profiling_settings=*/std::nullopt,
           /*ff_handle=*/device_handle,
           /*per_device_op_state=*/std::nullopt,
           /*optimizer_attrs=*/optimizer_attrs,
@@ -58,20 +56,15 @@ DynamicNodeInvocation
 DynamicOpenDataflowGraph perform_per_device_op_state_initialization(
     DynamicOpenDataflowGraph const &dg,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
     device_handle_t const &device_handle,
     OptimizerAttrs const &optimizer_attrs,
     global_device_id_t device_idx) {
 
   ASSERT(no_nodes_are_initialized(dg));
   DynamicOpenDataflowGraph result = transform_dynamic_invocation_set(
       dg, [&](DynamicNodeInvocation const &invocation) {
-        return initialize_node(invocation,
-                               allocator,
-                               profiling_settings,
-                               device_handle,
-                               optimizer_attrs,
-                               device_idx);
+        return initialize_node(
+            invocation, allocator, device_handle, optimizer_attrs, device_idx);
       });
 
   return result;

lib/local-execution/src/local-execution/task_execution.cc

@@ -45,7 +45,7 @@ TaskTensorParameter make_task_tensor_parameter_from_dynamic_slot(
 TaskArgumentAccessor make_task_argument_accessor_for_invocation(
     DynamicNodeInvocation const &invocation,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
+    std::optional<ProfilingSettings> const &profiling_settings,
     device_handle_t const &ff_handle,
     std::optional<PerDeviceOpState> const &per_device_op_state,
     std::optional<OptimizerAttrs> const &optimizer_attrs,
@@ -84,7 +84,7 @@ TaskArgumentAccessor make_task_argument_accessor_for_invocation(
 std::optional<milliseconds_t> execute_dynamic_node_invocation(
     DynamicNodeInvocation const &invocation,
     Allocator &allocator,
-    ProfilingSettings const &profiling_settings,
+    std::optional<ProfilingSettings> const &profiling_settings,
     device_handle_t const &ff_handle,
     std::optional<PerDeviceOpState> const &per_device_op_state,
     std::optional<OptimizerAttrs> const &optimizer_attrs,

lib/local-execution/test/src/local-execution/computation_graph_instance.cc

@@ -161,7 +161,6 @@ TEST_SUITE(FF_TEST_SUITE) {
             },
             /*input_tensors=*/input_tensors,
             /*allocator=*/allocator,
-            /*profiling_settings=*/ProfilingSettings{0_n, 1_p},
             /*device_handle=*/ff_handle,
             /*global_device_id=*/global_device_id);
 
@@ -335,7 +334,6 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) {
             },
             /*input_tensors=*/input_tensors,
             /*allocator=*/allocator,
-            /*profiling_settings=*/ProfilingSettings{0_n, 1_p},
             /*device_handle=*/ff_handle,
             /*device_idx=*/device_idx);
 
@@ -459,7 +457,6 @@ TEST_SUITE(FF_CUDA_TEST_SUITE) {
               },
               /*input_tensors=*/input_tensors,
               /*allocator=*/allocator,
-              /*profiling_settings=*/ProfilingSettings{0_n, 1_p},
               /*device_handle=*/ff_handle,
               /*device_idx=*/device_idx);
 

lib/realm-execution/include/realm-execution/distributed_per_device_op_state_initialization.h

@@ -1,7 +1,6 @@
 #ifndef _FLEXFLOW_LIB_REALM_EXECUTION_INCLUDE_REALM_EXECUTION_DISTRIBUTED_PER_DEVICE_OP_STATE_INITIALIZATION_H
 #define _FLEXFLOW_LIB_REALM_EXECUTION_INCLUDE_REALM_EXECUTION_DISTRIBUTED_PER_DEVICE_OP_STATE_INITIALIZATION_H
 
-#include "kernels/profiling_settings.dtg.h"
 #include "pcg/optimizer_attrs.dtg.h"
 #include "realm-execution/distributed_ff_handle.h"
 #include "realm-execution/per_device_op_state_backing.dtg.h"
@@ -22,7 +21,6 @@ PerDeviceOpStateBacking perform_distributed_per_device_op_state_initialization(
     RealmContext &ctx,
     DynamicOpenDataflowGraph const &dg,
     TensorInstanceBacking const &tensor_instance_backing,
-    ProfilingSettings const &profiling_settings,
     DistributedFfHandle const &device_handle,
     OptimizerAttrs const &optimizer_attrs,
     Realm::Event precondition);

lib/realm-execution/include/realm-execution/pcg_instance.h

@@ -3,7 +3,6 @@
 
 #include "kernels/allocation.h"
 #include "kernels/device_handle_t.dtg.h"
-#include "kernels/profiling_settings.dtg.h"
 #include "pcg/mapped_parallel_computation_graph/mapped_parallel_computation_graph.dtg.h"
 #include "pcg/optimizer_attrs.dtg.h"
 #include "realm-execution/distributed_ff_handle.h"
@@ -84,7 +83,6 @@ PCGInstance create_pcg_instance(
     OptimizerAttrs const &optimizer_attrs,
     std::optional<ParallelLossConfig> const &loss,
     std::map<DynamicValueAttrs, DynamicTensorAccessor> const &input_tensors,
-    ProfilingSettings const &profiling_settings,
     DistributedFfHandle const &ff_handle,
     DeviceType device_type);
 
@@ -100,28 +98,20 @@ PCGInstance create_pcg_instance(
  * \relates PCGInstance
  */
 std::map<dynamic_layer_guid_t, Realm::Event>
-    perform_all_passes_for_pcg_instance(
-        PCGInstance &pcg_instance,
-        ProfilingSettings const &profiling_settings,
-        DistributedFfHandle const &ff_handle);
+    perform_all_passes_for_pcg_instance(PCGInstance &pcg_instance,
+                                        DistributedFfHandle const &ff_handle);
 
 std::map<dynamic_layer_guid_t, Realm::Event>
-    perform_forward_pass_for_pcg_instance(
-        PCGInstance &pcg_instance,
-        ProfilingSettings const &profiling_settings,
-        DistributedFfHandle const &ff_handle);
+    perform_forward_pass_for_pcg_instance(PCGInstance &pcg_instance,
+                                          DistributedFfHandle const &ff_handle);
 
 std::map<dynamic_layer_guid_t, Realm::Event>
     perform_backward_pass_for_pcg_instance(
-        PCGInstance &pcg_instance,
-        ProfilingSettings const &profiling_settings,
-        DistributedFfHandle const &ff_handle);
+        PCGInstance &pcg_instance, DistributedFfHandle const &ff_handle);
 
 std::map<dynamic_layer_guid_t, Realm::Event>
-    perform_update_pass_for_pcg_instance(
-        PCGInstance &pcg_instance,
-        ProfilingSettings const &profiling_settings,
-        DistributedFfHandle const &ff_handle);
+    perform_update_pass_for_pcg_instance(PCGInstance &pcg_instance,
+                                         DistributedFfHandle const &ff_handle);
 
 } // namespace FlexFlow
 

lib/realm-execution/include/realm-execution/tasks/impl/op_task.h

@@ -1,7 +1,6 @@
 #ifndef _FLEXFLOW_LIB_REALM_EXECUTION_INCLUDE_REALM_EXECUTION_TASKS_IMPL_OP_TASK_H
 #define _FLEXFLOW_LIB_REALM_EXECUTION_INCLUDE_REALM_EXECUTION_TASKS_IMPL_OP_TASK_H
 
-#include "kernels/profiling_settings.dtg.h"
 #include "op-attrs/ops/loss_functions/loss_attrs.dtg.h"
 #include "pcg/optimizer_attrs.dtg.h"
 #include "realm-execution/device_specific_managed_per_device_ff_handle.h"
@@ -57,7 +56,6 @@ Realm::Event spawn_op_task(
     DynamicNodeInvocation const &invocation,
     TensorInstanceBacking const &tensor_backing,
     std::optional<DeviceSpecificPtr<PerDeviceOpState>> const &device_state,
-    ProfilingSettings const &profiling_settings,
     DeviceSpecificPtr<ManagedPerDeviceFFHandle> const &device_handle,
     std::optional<OptimizerAttrs> const &optimizer_attrs,
     Realm::Event precondition);