Skip to content

Commit 35d6199

Browse files
sawenzelclaude
andcommitted
Set the static TLS reserve for Geant4 inside the sim executables
This fixes o2-sim on the GRID, where Geant4 cannot be loaded since it uses the initial-exec TLS model. - A Geant4 loaded by dlopen needs a glibc static TLS reserve, which the Geant4 environment sets in GLIBC_TUNABLES. - In GRID jobs the variable arrives truncated as glibc.rtld.optional_static_tls, without its value, and is ignored. - The executables that load Geant4 now check the reserve first in main and, if it is missing, malformed or too small, restart themselves once with it set. - Other tunables are kept, and a larger reserve set by the user is left as is. https://its.cern.ch/jira/browse/O2-7305 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent fb508a3 commit 35d6199

5 files changed

Lines changed: 93 additions & 0 deletions

File tree

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
#ifndef O2_SIMSETUP_STATICTLSRESERVE_H
13+
#define O2_SIMSETUP_STATICTLSRESERVE_H
14+
15+
#include <cstdlib>
16+
#include <string>
17+
#include <string_view>
18+
#include <unistd.h>
19+
20+
namespace o2
21+
{
22+
23+
/// The GLIBC_TUNABLES value with a static TLS reserve of at least `reserve` bytes, or an empty string if the
24+
/// given value already has one. Other tunables are kept; a malformed or smaller reserve is replaced.
25+
inline std::string withStaticTLSReserve(const char* tunables, unsigned long reserve)
26+
{
27+
constexpr std::string_view name = "glibc.rtld.optional_static_tls";
28+
std::string kept;
29+
bool sufficient = false;
30+
std::string_view rest = tunables ? tunables : "";
31+
while (!rest.empty()) {
32+
const auto colon = rest.find(':');
33+
const std::string_view entry = rest.substr(0, colon);
34+
rest = colon == std::string_view::npos ? std::string_view{} : rest.substr(colon + 1);
35+
if (entry.substr(0, name.size()) == name) {
36+
if (entry.size() > name.size() + 1 && entry[name.size()] == '=') {
37+
const std::string value(entry.substr(name.size() + 1));
38+
char* end = nullptr;
39+
const unsigned long bytes = std::strtoul(value.c_str(), &end, 0);
40+
if (end && *end == '\0' && bytes >= reserve) {
41+
sufficient = true;
42+
break;
43+
}
44+
}
45+
continue; // drop a reserve that is malformed or too small
46+
}
47+
if (!entry.empty()) {
48+
kept += (kept.empty() ? "" : ":") + std::string(entry);
49+
}
50+
}
51+
if (sufficient) {
52+
return {};
53+
}
54+
return std::string(name) + "=" + std::to_string(reserve) + (kept.empty() ? "" : ":" + kept);
55+
}
56+
57+
/// Restarts the program once with a glibc static TLS reserve, which a Geant4 built with the initial-exec TLS
58+
/// model needs when it is loaded by dlopen. glibc reads GLIBC_TUNABLES only at program start, so the reserve
59+
/// cannot be set later. Call this first in main, before any thread is started.
60+
inline void ensureStaticTLSReserve(char* argv[], unsigned long reserve = 32768)
61+
{
62+
#ifdef __linux__
63+
constexpr const char* guard = "O2_STATIC_TLS_REEXEC";
64+
if (getenv(guard)) {
65+
unsetenv(guard); // already restarted once; do not loop if glibc ignores the tunable
66+
return;
67+
}
68+
const auto tunables = withStaticTLSReserve(getenv("GLIBC_TUNABLES"), reserve);
69+
if (tunables.empty()) {
70+
return;
71+
}
72+
setenv("GLIBC_TUNABLES", tunables.c_str(), 1);
73+
setenv(guard, "1", 1);
74+
execv("/proc/self/exe", argv);
75+
// if the restart fails we carry on; a failing dlopen reports the cause
76+
#endif
77+
}
78+
79+
} // namespace o2
80+
81+
#endif

‎run/O2SimDeviceRunner.cxx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
#include "rapidjson/document.h"
4040
#include "rapidjson/stringbuffer.h"
4141
#include "rapidjson/filereadstream.h"
42+
#include <SimSetup/StaticTLSReserve.h>
4243
namespace bpo = boost::program_options;
4344

4445
std::vector<int> gChildProcesses; // global vector of child pids
@@ -267,6 +268,8 @@ bool waitForControlInput(int workerID)
267268

268269
int main(int argc, char* argv[])
269270
{
271+
// Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start
272+
o2::ensureStaticTLSReserve(argv);
270273
struct sigaction act;
271274
memset(&act, 0, sizeof act);
272275
sigemptyset(&act.sa_mask);

‎run/o2sim.cxx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@
1313
#include <SimConfig/SimConfig.h>
1414
#include <TStopwatch.h>
1515
#include <fairlogger/Logger.h>
16+
#include <SimSetup/StaticTLSReserve.h>
1617

1718
int main(int argc, char* argv[])
1819
{
20+
// Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start
21+
o2::ensureStaticTLSReserve(argv);
1922
TStopwatch timer;
2023
timer.Start();
2124
auto& conf = o2::conf::SimConfig::Instance();

‎run/o2sim_evalmat.cxx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,12 @@
1414
#include <SimConfig/MatMapParams.h>
1515
#include <TStopwatch.h>
1616
#include <fairlogger/Logger.h>
17+
#include <SimSetup/StaticTLSReserve.h>
1718

1819
int main(int argc, char* argv[])
1920
{
21+
// Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start
22+
o2::ensureStaticTLSReserve(argv);
2023
TStopwatch timer;
2124
timer.Start();
2225

‎run/o2sim_mbudget.cxx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@
1313
#include <SimConfig/SimConfig.h>
1414
#include <TStopwatch.h>
1515
#include <fairlogger/Logger.h>
16+
#include <SimSetup/StaticTLSReserve.h>
1617

1718
int main(int argc, char* argv[])
1819
{
20+
// Geant4 is loaded by dlopen and may need a static TLS reserve, which glibc only takes at program start
21+
o2::ensureStaticTLSReserve(argv);
1922
TStopwatch timer;
2023
timer.Start();
2124
auto& conf = o2::conf::SimConfig::Instance();

0 commit comments

Comments
 (0)