DPL: allow building a workflow as a workflow plugin

A workflow opts in with WORKFLOW_PLUGIN, on o2_add_dpl_workflow or on
o2_add_executable. Its sources are then compiled once, into the DSO
libO2<Name>Plugin holding the workflow plugin <Name>, the CamelCase of
the executable name without o2-: o2-testworkflows-diamond-workflow gives
O2TestworkflowsDiamondWorkflowPlugin:TestworkflowsDiamondWorkflow. The
executable keeps its name and options, and only has a main() which
calls dpl_workflow_main() of the plugin it links. That one runs the
plugin as the one embedded in the executable, exactly as main() does for
any other workflow, so its behaviour and dumps are the same. The plugin
is installed in lib with the executable.

WORKFLOW_PLUGIN, rather than PLUGIN, because DPL has several kinds of
plugins and this one is DplPluginKind::Workflow, loaded with
--workflow-plugin; the same name works on both functions.

The existing o2-dpl-run, whose embedded workflow is empty, is the
generic loader: o2-dpl-run --workflow-plugin A:X,B:Y runs the workflows
of the plugins, and its children are again o2-dpl-run.

Workflows which do not opt in are built exactly as before. The diamond
workflow, the simple source, processor and sink, and the two test
plugins of test_WorkflowPlugins opt in.

test_Framework_test_WorkflowPluginDump checks that the plugin of the
diamond workflow, run by o2-dpl-run, defines the same workflow as its
executable, apart from the executable, plugin and command line.

Author: ktf

Framework/Core/CMakeLists.txt

@@ -459,18 +459,16 @@ o2_add_test(
   TARGETVARNAME workflowPluginsTest
   NO_BOOST_TEST
   COMMAND_LINE_ARGS
-    --workflow-plugin O2FrameworkTestWorkflowPluginA:TestWorkflowPluginA,O2FrameworkTestWorkflowPluginB:TestWorkflowPluginB
+    --workflow-plugin O2TestFrameworkWorkflowPluginAPlugin:TestFrameworkWorkflowPluginA,O2TestFrameworkWorkflowPluginBPlugin:TestFrameworkWorkflowPluginB
     --shared 3 --run --shm-segment-size 20000000 ${DPL_WORKFLOW_TESTS_EXTRA_OPTIONS}
   ENVIRONMENT "DYLD_LIBRARY_PATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}:$ENV{DYLD_LIBRARY_PATH};LD_LIBRARY_PATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}:$ENV{LD_LIBRARY_PATH}"
   )
 if(workflowPluginsTest)
   foreach(p A B)
-    add_library(O2FrameworkTestWorkflowPlugin${p} SHARED test/test_WorkflowPlugin${p}.cxx)
-    target_link_libraries(O2FrameworkTestWorkflowPlugin${p} PRIVATE O2::Framework)
-    target_compile_definitions(O2FrameworkTestWorkflowPlugin${p} PRIVATE
-                               DPL_WORKFLOW_PLUGIN_NAME=TestWorkflowPlugin${p}
-                               DPL_WORKFLOW_PLUGIN_LIBRARY=O2FrameworkTestWorkflowPlugin${p})
-    add_dependencies(${workflowPluginsTest} O2FrameworkTestWorkflowPlugin${p})
+    o2_add_executable(WorkflowPlugin${p} SOURCES test/test_WorkflowPlugin${p}.cxx
+                      COMPONENT_NAME Framework PUBLIC_LINK_LIBRARIES O2::Framework
+                      IS_TEST NO_INSTALL WORKFLOW_PLUGIN TARGETVARNAME workflowPlugin)
+    add_dependencies(${workflowPluginsTest} ${workflowPlugin})
   endforeach()
   add_test(NAME test_Framework_test_WorkflowPluginsDump
            COMMAND ${CMAKE_CURRENT_LIST_DIR}/test/test_WorkflowPluginsDump.sh $<TARGET_FILE:${workflowPluginsTest}>)

Framework/Core/include/Framework/runDataProcessing.h

@@ -170,15 +170,12 @@ char* getIdString(int argc, char** argv);
 #define DPL_STRINGIZE(A) DPL_STRINGIZE_NX(A)
 
 // The workflow definition is a plugin. By default it is embedded in the
-// executable as DPL_WORKFLOW_PLUGIN_NAME, with an empty library name.
-// To build it in a DSO instead, both DPL_WORKFLOW_PLUGIN_NAME and
-// DPL_WORKFLOW_PLUGIN_LIBRARY must be specified at build time.
+// executable as DPL_WORKFLOW_PLUGIN_NAME, and main() runs it. A workflow built
+// as a plugin library gets DPL_WORKFLOW_PLUGIN_NAME at build time: it has no
+// main(), and the executable which links it calls dpl_workflow_main() instead.
 #ifndef DPL_WORKFLOW_PLUGIN_NAME
 #define DPL_WORKFLOW_PLUGIN_NAME InternalCustomWorkflow
-#ifdef DPL_WORKFLOW_PLUGIN_LIBRARY
-#error Missing DPL_WORKFLOW_PLUGIN_NAME
-#endif
-#define DPL_WORKFLOW_PLUGIN_LIBRARY
+#define DPL_WORKFLOW_PLUGIN_EMBEDDED
 #endif
 
 struct DPL_WORKFLOW_PLUGIN_NAME : o2::framework::WorkflowPlugin {
@@ -216,11 +213,15 @@ DPLPluginHandle* dpl_plugin_callback(DPLPluginHandle* previous)
 // An empty library means the plugin embedded in the executable, found via @a embeddedPlugins.
 int callMain(int argc, char** argv, char const* pluginSpec, DPLPluginHandle* (*embeddedPlugins)(DPLPluginHandle*));
 
+#ifdef DPL_WORKFLOW_PLUGIN_EMBEDDED
 int main(int argc, char** argv)
+#else
+extern "C" int dpl_workflow_main(int argc, char** argv)
+#endif
 {
   using namespace o2::framework;
 
-  int result = callMain(argc, argv, DPL_STRINGIZE(DPL_WORKFLOW_PLUGIN_LIBRARY) ":" DPL_STRINGIZE(DPL_WORKFLOW_PLUGIN_NAME), dpl_plugin_callback);
+  int result = callMain(argc, argv, ":" DPL_STRINGIZE(DPL_WORKFLOW_PLUGIN_NAME), dpl_plugin_callback);
 
   char* idstring = getIdString(argc, argv);
   o2::framework::OnWorkflowTerminationHook onWorkflowTerminationHook;

Framework/Core/test/test_WorkflowPluginsDump.sh

@@ -2,8 +2,8 @@
 # Dumps the workflow of the two test plugins and checks that it records,
 # for each data processor, the plugin which defines it.
 # usage: test_WorkflowPluginsDump.sh <o2-test-framework-WorkflowPlugins>
-A=O2FrameworkTestWorkflowPluginA:TestWorkflowPluginA
-B=O2FrameworkTestWorkflowPluginB:TestWorkflowPluginB
+A=O2TestFrameworkWorkflowPluginAPlugin:TestFrameworkWorkflowPluginA
+B=O2TestFrameworkWorkflowPluginBPlugin:TestFrameworkWorkflowPluginB
 dump=$("$1" -b --workflow-plugin $A,$B --dump-workflow < /dev/null)
 echo "$dump" | grep -q "\"plugin\": \"$A\""
 echo "$dump" | grep -q "\"plugin\": \"$B\""

Framework/TestWorkflows/CMakeLists.txt

@@ -31,7 +31,8 @@ o2_add_dpl_workflow(o2rootmessage-workflow
 
 o2_add_dpl_workflow(diamond-workflow
                   SOURCES src/o2DiamondWorkflow.cxx
-                  COMPONENT_NAME TestWorkflows)
+                  COMPONENT_NAME TestWorkflows
+                  WORKFLOW_PLUGIN)
 
 o2_add_dpl_workflow(dummy-populator-workflow
                   SOURCES src/o2DummyPopulatorWorkflow.cxx
@@ -114,7 +115,8 @@ o2_add_dpl_workflow(ccdb-fetch-to-timeframe
 
 o2_add_dpl_workflow(simple-source
                   SOURCES src/o2SimpleSource.cxx
-                  COMPONENT_NAME TestWorkflows)
+                  COMPONENT_NAME TestWorkflows
+                  WORKFLOW_PLUGIN)
 
 o2_add_dpl_workflow(simple-fairmq-source
                   SOURCES src/o2SimpleFairMQSource.cxx
@@ -126,11 +128,13 @@ o2_add_dpl_workflow(simple-fairmq-sink
 
 o2_add_dpl_workflow(simple-sink
                   SOURCES src/o2SimpleSink.cxx
-                  COMPONENT_NAME TestWorkflows)
+                  COMPONENT_NAME TestWorkflows
+                  WORKFLOW_PLUGIN)
 
 o2_add_dpl_workflow(simple-processor
                   SOURCES src/o2SimpleProcessor.cxx
-                  COMPONENT_NAME TestWorkflows)
+                  COMPONENT_NAME TestWorkflows
+                  WORKFLOW_PLUGIN)
 
 o2_add_dpl_workflow(analysis-workflow
                   SOURCES src/o2AnalysisWorkflow.cxx
@@ -156,3 +160,13 @@ o2_add_executable(fairmq-header-size-test
 add_executable(o2-test-deadlock src/o2DeadlockReproducer.cxx)
 target_link_libraries(o2-test-deadlock PUBLIC FairMQ::FairMQ)
 add_test(NAME o2-test-deadlock COMMAND o2-test-deadlock --channel-config "name=data,type=sub,method=bind,address=ipc://127.0.0.1,rateLogging=0" --id foo --transport zeromq --control static)
+
+if(BUILD_TESTING)
+  add_test(NAME test_Framework_test_WorkflowPluginDump
+           COMMAND ${CMAKE_CURRENT_LIST_DIR}/test/test_WorkflowPluginDump.sh
+                   $<TARGET_FILE:O2exe-testworkflows-diamond-workflow> $<TARGET_FILE:O2exe-dpl-run>
+                   O2TestworkflowsDiamondWorkflowPlugin:TestworkflowsDiamondWorkflow)
+  set_property(TEST test_Framework_test_WorkflowPluginDump PROPERTY LABELS framework)
+  set_property(TEST test_Framework_test_WorkflowPluginDump PROPERTY ENVIRONMENT
+               "LD_LIBRARY_PATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}:$ENV{LD_LIBRARY_PATH}")
+endif()

Framework/TestWorkflows/test/test_WorkflowPluginDump.sh

@@ -0,0 +1,10 @@
+#!/bin/sh -e
+# Checks that the workflow plugin of an executable, run by o2-dpl-run,
+# defines the same workflow as the executable itself.
+# usage: test_WorkflowPluginDump.sh <executable> <o2-dpl-run> <library>:<plugin>
+strip() { grep -v '"executable"\|"plugin"\|"command"\|"--workflow-plugin"\|"'"$1"'"'; }
+exe=$("$1" -b --dump-workflow < /dev/null | strip "$3")
+dso=$("$2" -b --workflow-plugin "$3" --dump-workflow < /dev/null | strip "$3")
+test -n "$exe"
+test "$exe" = "$dso"
+test "$("$2" -b --workflow-plugin "$3" --dump-workflow < /dev/null | grep -c "\"plugin\": \"$3\"")" -gt 0

cmake/O2AddExecutable.cmake

@@ -30,6 +30,11 @@ include_guard()
 #   regular executables (e.g. o2-tpc-... or o2-mch-...) or the origin target for
 #   tests (to help locate the source file in the source directory hierarchy,
 #   e.g. o2-test-DataFormats-...)
+# * WORKFLOW_PLUGIN (optional), for a DPL workflow: its sources are compiled
+#   into the workflow plugin <Name>, in libO2<Name>Plugin, which o2-dpl-run can
+#   load with --workflow-plugin, and the executable is only a main() linking
+#   it. <Name> is the CamelCase of the executable name without o2-, e.g.
+#   o2-tpc-reco-workflow gives O2TpcRecoWorkflowPlugin:TpcRecoWorkflow.
 #
 # Note that the _target_ corresponding to the executable will be named
 # O2exe[-exeType][component]-basename and can be retrieved with the
@@ -48,7 +53,7 @@ function(o2_add_executable baseTargetName)
   cmake_parse_arguments(PARSE_ARGV
                         1
                         A
-                        "IS_TEST;NO_INSTALL;IS_BENCHMARK"
+                        "IS_TEST;NO_INSTALL;IS_BENCHMARK;WORKFLOW_PLUGIN"
                         "COMPONENT_NAME;TARGETVARNAME"
                         "SOURCES;PUBLIC_LINK_LIBRARIES;JOB_POOL")
 
@@ -92,7 +97,18 @@ function(o2_add_executable baseTargetName)
   endif()
 
   # add the executable with its sources
-  add_executable(${target} ${A_SOURCES})
+  if(A_WORKFLOW_PLUGIN)
+    o2_add_workflow_plugin(${exeName} SOURCES ${A_SOURCES}
+                           PUBLIC_LINK_LIBRARIES ${A_PUBLIC_LINK_LIBRARIES}
+                           TARGETVARNAME plugin)
+    add_executable(${target} $<TARGET_OBJECTS:O2FrameworkWorkflowMain>)
+    target_link_libraries(${target} PUBLIC ${plugin})
+    if(NOT A_NO_INSTALL)
+      install(TARGETS ${plugin} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+    endif()
+  else()
+    add_executable(${target} ${A_SOURCES})
+  endif()
 
   # set the executable output name
   set_property(TARGET ${target} PROPERTY OUTPUT_NAME ${exeName})
@@ -132,3 +148,32 @@ function(o2_add_executable baseTargetName)
   endif()
 
 endfunction()
+
+# o2_add_workflow_plugin(exeName SOURCES ... PUBLIC_LINK_LIBRARIES ...
+# TARGETVARNAME var) builds the workflow plugin of the executable exeName, see
+# WORKFLOW_PLUGIN above, and the main() of such executables, once.
+function(o2_add_workflow_plugin exeName)
+  cmake_parse_arguments(PARSE_ARGV 1 A "" "TARGETVARNAME" "SOURCES;PUBLIC_LINK_LIBRARIES")
+  string(REGEX REPLACE "^o2-" "" name ${exeName})
+  string(REPLACE "-" ";" parts ${name})
+  set(name "")
+  foreach(part IN LISTS parts)
+    string(SUBSTRING ${part} 0 1 first)
+    string(SUBSTRING ${part} 1 -1 rest)
+    string(TOUPPER ${first} first)
+    string(APPEND name ${first}${rest})
+  endforeach()
+  string(MAKE_C_IDENTIFIER ${name} name)
+  set(plugin O2${name}Plugin)
+  add_library(${plugin} SHARED ${A_SOURCES})
+  target_link_libraries(${plugin} PUBLIC ${A_PUBLIC_LINK_LIBRARIES})
+  target_compile_definitions(${plugin} PRIVATE DPL_WORKFLOW_PLUGIN_NAME=${name})
+  set(${A_TARGETVARNAME} ${plugin} PARENT_SCOPE)
+  if(NOT TARGET O2FrameworkWorkflowMain)
+    file(CONFIGURE OUTPUT ${CMAKE_BINARY_DIR}/O2FrameworkWorkflowMain.cxx CONTENT [[
+extern "C" int dpl_workflow_main(int argc, char** argv);
+int main(int argc, char** argv) { return dpl_workflow_main(argc, argv); }
+]])
+    add_library(O2FrameworkWorkflowMain OBJECT ${CMAKE_BINARY_DIR}/O2FrameworkWorkflowMain.cxx)
+  endif()
+endfunction()

cmake/O2AddWorkflow.cmake

@@ -17,7 +17,8 @@ include_guard()
 # file via <executable-name> --dump so that it can be easily registered and
 # deployed e.g. in the train infrastructure.
 #
-# For most of the options please see how o2_add_executable works.
+# For most of the options, e.g. WORKFLOW_PLUGIN, please see how
+# o2_add_executable works.
 #
 # The installed executable will be named as regular executables (see
 # o2_add_executable for details)
@@ -28,14 +29,17 @@ include_guard()
 
 function(o2_add_dpl_workflow baseTargetName)
 
-  cmake_parse_arguments(PARSE_ARGV 1 A "" "COMPONENT_NAME;TARGETVARNAME"
+  cmake_parse_arguments(PARSE_ARGV 1 A "WORKFLOW_PLUGIN" "COMPONENT_NAME;TARGETVARNAME"
                         "SOURCES;PUBLIC_LINK_LIBRARIES;JOB_POOL")
 
   if(A_UNPARSED_ARGUMENTS)
     message(FATAL_ERROR "Got trailing arguments ${A_UNPARSED_ARGUMENTS}")
   endif()
 
-  o2_add_executable(${baseTargetName}
+  if(A_WORKFLOW_PLUGIN)
+    set(workflowPlugin WORKFLOW_PLUGIN)
+  endif()
+  o2_add_executable(${baseTargetName} ${workflowPlugin}
     COMPONENT_NAME ${A_COMPONENT_NAME} TARGETVARNAME targetExeName
     SOURCES ${A_SOURCES}
     PUBLIC_LINK_LIBRARIES O2::Framework ${A_PUBLIC_LINK_LIBRARIES})