From 771dcdd567a44706b3d87f35fda25864c009c9f0 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse Date: Fri, 20 Mar 2020 21:26:47 +0100 Subject: [PATCH] DPL: do own management of shared memory We do not spawn a shm-monitor because that will create zombies when launched inside a docker container. Instead by default we cleanup the shared memory session on exit. Default behavior can still be obtained by passing --shm-monitor true. --- Framework/Core/src/DeviceSpecHelpers.cxx | 3 +++ Framework/Core/src/runDataProcessing.cxx | 19 ++++++++++++++++++- .../Core/test/test_FrameworkDataFlowToDDS.cxx | 8 ++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Framework/Core/src/DeviceSpecHelpers.cxx b/Framework/Core/src/DeviceSpecHelpers.cxx index 6f7c7b721ea91..64af9a47c293c 100644 --- a/Framework/Core/src/DeviceSpecHelpers.cxx +++ b/Framework/Core/src/DeviceSpecHelpers.cxx @@ -801,6 +801,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped, std::vector tmpArgs = {argv[0], "--id", spec.id.c_str(), "--control", "static", + "--shm-monitor", "false", "--log-color", "false", "--color", "false"}; if (defaultStopped) { @@ -850,6 +851,7 @@ void DeviceSpecHelpers::prepareArguments(bool defaultQuiet, bool defaultStopped, realOdesc.add_options()("child-driver", bpo::value()); realOdesc.add_options()("rate", bpo::value()); realOdesc.add_options()("shm-segment-size", bpo::value()); + realOdesc.add_options()("shm-monitor", bpo::value()); realOdesc.add_options()("session", bpo::value()); filterArgsFct(expansions.we_wordc, expansions.we_wordv, realOdesc); wordfree(&expansions); @@ -949,6 +951,7 @@ boost::program_options::options_description DeviceSpecHelpers::getForwardedDevic ("plugin-search-path,S", bpo::value(), "FairMQ plugins search path") // ("control-port", bpo::value(), "Utility port to be used by O2 Control") // ("rate", bpo::value(), "rate for a data source device (Hz)") // + ("shm-monitor", bpo::value(), "whether to use the shared memory monitor") // ("shm-segment-size", bpo::value(), "size of the shared memory segment in bytes") // ("session", bpo::value(), "unique label for the shared memory session") // ("monitoring-backend", bpo::value(), "monitoring connection string") // diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx index fc111d9c90084..eebd10e5ed252 100644 --- a/Framework/Core/src/runDataProcessing.cxx +++ b/Framework/Core/src/runDataProcessing.cxx @@ -273,6 +273,18 @@ static void handle_sigint(int) } } +/// Helper to invoke shared memory cleanup +void cleanupSHM(std::string const& uniqueWorkflowId) +{ + auto shmCleanup = fmt::format("fairmq-shmmonitor --cleanup -s dpl_{} 2>&1 >/dev/null", uniqueWorkflowId); + LOG(debug) + << "Cleaning up shm memory session with " << shmCleanup; + auto result = system(shmCleanup.c_str()); + if (result != 0) { + LOG(error) << "Unable to cleanup shared memory, run " << shmCleanup << "by hand to fix"; + } +} + static void handle_sigchld(int) { sigchld_requested = true; } void spawnRemoteDevice(std::string const& forwardedStdin, @@ -848,6 +860,9 @@ int runStateMachine(DataProcessorSpecs const& workflow, } FD_ZERO(&(driverInfo.childFdset)); + /// Cleanup the shared memory for the uniqueWorkflowId, in + /// case we are unlucky and an old one is already present. + cleanupSHM(driverInfo.uniqueWorkflowId); /// After INIT we go into RUNNING and eventually to SCHEDULE from /// there and back into running. This is because the general case /// would be that we start an application and then we wait for @@ -1057,8 +1072,10 @@ int runStateMachine(DataProcessorSpecs const& workflow, driverInfo.states.push_back(DriverState::GUI); } break; - case DriverState::EXIT: + case DriverState::EXIT: { + cleanupSHM(driverInfo.uniqueWorkflowId); return calculateExitCode(infos); + } case DriverState::PERFORM_CALLBACKS: for (auto& callback : driverControl.callbacks) { callback(workflow, deviceSpecs, deviceExecutions, dataProcessorInfos); diff --git a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx index 4cbdf2668df45..c450efea66d80 100644 --- a/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx +++ b/Framework/Core/test/test_FrameworkDataFlowToDDS.cxx @@ -97,16 +97,16 @@ BOOST_AUTO_TEST_CASE(TestDDS) dumpDeviceSpec2DDS(ss, devices, executions); BOOST_CHECK_EQUAL(ss.str(), R"EXPECTED( - foo --id A --control static --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds + foo --id A --control static --shm-monitor false --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds - foo --id B --control static --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds + foo --id B --control static --shm-monitor false --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds - foo --id C --control static --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds + foo --id C --control static --shm-monitor false --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds - foo --id D --control static --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds + foo --id D --control static --shm-monitor false --log-color false --color false --jobs 4 --session dpl_workflow-id --plugin-search-path $FAIRMQ_ROOT/lib --plugin dds