LArSoft  v09_90_00
Liquid Argon Software toolkit - https://larsoft.org/
run_art.cc
Go to the documentation of this file.
2 // vim: set sw=2 expandtab :
3 
15 #include "boost/program_options.hpp"
17 #include "cetlib/HorizontalRule.h"
18 #include "cetlib/container_algorithms.h"
19 #include "cetlib/ostream_handle.h"
20 #include "cetlib_except/exception.h"
21 #include "fhiclcpp/ParameterSet.h"
25 #include "fhiclcpp/parse.h"
27 
28 #include <cassert>
29 #include <exception>
30 #include <iostream>
31 #include <string>
32 
33 #ifdef __linux__
34 #include <malloc.h>
35 #endif // __linux__
36 
37 using namespace std;
38 using namespace string_literals;
39 
41 
42 namespace {
43 
44  cet::ostream_handle
45  make_ostream_handle(std::string const& filename)
46  {
47  assert(!filename.empty());
48  if (art::detail::output_to_stderr(filename)) {
49  return cet::ostream_handle{std::cerr};
50  } else if (art::detail::output_to_stdout(filename)) {
51  return cet::ostream_handle{std::cout};
52  } else {
53  auto os = cet::ostream_handle{filename};
54  if (!os) {
56  << "Unable to write post-processed configuration to specified file "
57  << filename << ".\n";
58  }
59  return os;
60  }
61  }
62 
64  get_print_mode(std::string const& mode)
65  {
66  if (mode == "raw") {
68  } else if (mode == "annotate") {
70  } else if (mode == "prefix-annotate") {
72  }
74  << "Unrecognized ParameterSet printing mode: " << mode << '\n';
75  }
76 
77  std::string
78  banner(std::string const& filename)
79  {
80  std::string result = "** Config output ";
81  bool const to_terminal = art::detail::output_to_stderr(filename) ||
83  result +=
84  to_terminal ? "follows" : std::string("to file '" + filename + "'");
85  result += " **\n";
86  return result;
87  }
88 
89  void
90  print_config(fhicl::ParameterSet const& main_pset,
91  std::string const& output_file,
92  std::string const& mode)
93  {
94  std::cerr << banner(output_file);
95  auto os = make_ostream_handle(output_file);
96  os << main_pset.to_indented_string(0, get_print_mode(mode));
97  }
98 
99  enum class debug_processing : std::size_t {
100  config_summary,
101  config_out,
102  debug_config,
103  validate_config,
104  none
105  };
106 
108  debug_processing_mode(fhicl::ParameterSet const& scheduler_pset)
109  {
110  if (not scheduler_pset.has_key("debug")) {
111  return debug_processing::none;
112  }
113 
114  auto const processing_options = {
115  "config-summary", "config-out", "debug-config", "validate-config"};
116 
117  auto const option = scheduler_pset.get<std::string>("debug.option");
118  auto pos = cet::find_in_all(processing_options, option);
119  if (pos == cend(processing_options)) {
120  throw art::Exception{
122  "An error was encountered while processing debugging options"}
123  << "The debugging option '" << option << "' is not supported.\n"
124  << "If you did not explicitly provide this value in your configuration "
125  "file, please contact artists@fnal.gov and report this error. "
126  "Otherwise,\n"
127  << "choose from 'configSummary', 'configOut', 'debugConfig', or "
128  "'validateConfig'.\n";
129  }
130 
131  auto const index = std::distance(cbegin(processing_options), pos);
132  return static_cast<debug_processing>(index);
133  }
134 
135 } // unnamed namespace
136 
137 namespace bpo = boost::program_options;
138 
139 namespace art {
140 
141  int
142  run_art(int argc,
143  char** argv,
144  bpo::options_description& all_desc,
145  OptionsHandlers&& handlers)
146  {
147  // This must be added separately: how to deal with any non-option arguments.
148  bpo::positional_options_description pd;
149  // A single non-option argument will be taken to be the source data file.
150  pd.add("source", -1);
151  // Parse the command line.
152  bpo::variables_map vm;
153  try {
154  bpo::store(bpo::command_line_parser(argc, argv)
155  .options(all_desc)
156  .style(bpo::command_line_style::default_style &
157  ~bpo::command_line_style::allow_guessing)
158  .positional(pd)
159  .run(),
160  vm);
161  bpo::notify(vm);
162  }
163  catch (bpo::error const& e) {
164  cerr << "Exception from command line processing in " << argv[0] << ": "
165  << e.what() << '\n';
166  return 88;
167  }
168  // Preliminary argument checking.
169  for (auto& handler : handlers) {
170  auto const result = handler->checkOptions(vm);
171  if (result != 0) {
172  return result;
173  }
174  }
175  // Processing of arguments and post-processing of config.
176  fhicl::intermediate_table raw_config;
177  for (auto& handler : handlers) {
178  auto const result = handler->processOptions(vm, raw_config);
179  if (result != 0) {
180  return result;
181  }
182  }
183 
184  // If configuration pruning has been enabled, remove unused module
185  // configurations.
187  auto const scheduler_key = fhicl_key("services", "scheduler");
188  auto enabled_modules = detail::EnabledModules::none();
189  assert(exists_outside_prolog(raw_config, scheduler_key));
190  try {
191  auto const pruneConfigKey = fhicl_key(scheduler_key, "pruneConfig");
192  auto const reportUnusedKey = fhicl_key(scheduler_key, "reportUnused");
193  assert(exists_outside_prolog(raw_config, pruneConfigKey));
194  assert(exists_outside_prolog(raw_config, reportUnusedKey));
195  bool const prune_config = raw_config.get<bool>(pruneConfigKey);
196  bool const report_unused = raw_config.get<bool>(reportUnusedKey);
197  enabled_modules = detail::prune_config_if_enabled(
198  prune_config, report_unused, raw_config);
199  }
200  catch (Exception const& e) {
201  printArtException(e, "art");
202  return e.returnCode();
203  }
204  catch (cet::exception const& e) {
205  printArtException(e, "art");
206  return 65;
207  }
208  catch (std::bad_alloc const& bda) {
209  printBadAllocException("art");
210  return 68;
211  }
212  catch (std::exception const& e) {
213  printStdException(e, "art");
214  return 66;
215  }
216  catch (...) {
217  printUnknownException("art");
218  return 67;
219  }
220 
221  //
222  // Make the parameter set from the intermediate table.
223  //
224  fhicl::ParameterSet main_pset;
225  try {
226  main_pset = fhicl::ParameterSet::make(raw_config);
227  }
228  catch (cet::exception const& e) {
229  constexpr cet::HorizontalRule rule{36};
230  cerr << "ERROR: Failed to create a parameter set from parsed "
231  "configuration with exception "
232  << e.what() << ".\n";
233  cerr << " Intermediate configuration state follows:\n"
234  << rule('-') << '\n'
235  << rule('-') << '\n';
236  for (auto const& [key, value] : raw_config) {
237  cerr << key << ": " << value.to_string() << '\n';
238  }
239  cerr << rule('-') << '\n' << rule('-') << '\n';
240  return 91;
241  }
242  // Main parameter set must be placed in registry manually.
243  try {
245  }
246  catch (...) {
247  cerr << "Uncaught exception while inserting main parameter set into "
248  "registry.\n";
249  throw;
250  }
251 
252 #ifdef __linux__
253  // Tell the system memory allocator to only use one arena: they
254  // are 64 MiB in size, and the default is 8 * num_of_cores. Using
255  // the default means that when using 40 threads we get 40 arenas,
256  // which means we have 40 * 64 MiB = 2560 MiB of virtual address
257  // space devoted to per-thread heaps!!!
258  mallopt(M_ARENA_MAX, 1);
259 #endif // __linux__
260 
261  auto const services_pset =
262  main_pset.get<fhicl::ParameterSet>("services", {});
263  auto const scheduler_pset =
264  services_pset.get<fhicl::ParameterSet>("scheduler", {});
265 
266  // Handle early configuration-debugging
267  auto const debug_mode = debug_processing_mode(scheduler_pset);
268  if (debug_mode != debug_processing::none) {
269  auto const debug_pset = scheduler_pset.get<fhicl::ParameterSet>("debug");
270  auto const filename = debug_pset.get<std::string>("fileName");
271  auto const mode = debug_pset.get<std::string>("printMode");
272 
273  switch (debug_mode) {
274  case debug_processing::validate_config: {
275  [[fallthrough]];
276  }
277  case debug_processing::config_out: {
278  print_config(main_pset, filename, mode);
279  break;
280  }
281  case debug_processing::debug_config: {
282  print_config(main_pset, filename, mode);
283  return detail::info_success();
284  }
285  case debug_processing::config_summary: {
286  detail::print_config_summary(main_pset, mode, enabled_modules);
287  return detail::info_success();
288  }
289  case debug_processing::none:
290  break;
291  }
292  }
293  //
294  // Start the messagefacility
295  //
296  mf::SetIteration("JobSetup");
297  try {
299  services_pset.get<fhicl::ParameterSet>("message", {}));
300  }
301  catch (cet::exception const& e) {
302  cerr << e.what() << '\n';
303  return 69;
304  }
305  catch (exception const& e) {
306  cerr << e.what() << '\n';
307  return 70;
308  }
309  catch (...) {
310  cerr << "Caught unknown exception while initializing the message "
311  "facility.\n";
312  return 71;
313  }
314  mf::LogInfo("MF_INIT_OK") << "Messagelogger initialization complete.";
315  //
316  // Start the EventProcessor
317  //
318  int rc{0};
319  try {
320  EventProcessor ep{std::move(main_pset), std::move(enabled_modules)};
321  // Behavior of validate_config is to validate FHiCL syntax *and*
322  // user-specified configurations of paths, modules, services,
323  // etc. It is thus possible that an exception thrown during
324  // construction of the EventProcessor object can have nothing to
325  // do with a configuration error.
326  if (debug_mode == debug_processing::validate_config) {
327  return detail::info_success();
328  }
329  if (scheduler_pset.has_key("dataDependencyGraph")) {
330  return detail::info_success();
331  }
332  auto ep_rc = ep.runToCompletion();
333  if (ep_rc == EventProcessor::epSignal) {
334  cerr << "Art has handled signal " << art::shutdown_flag << ".\n";
335  if (scheduler_pset.get<bool>("errorOnSIGINT")) {
336  rc = 128 + art::shutdown_flag;
337  }
338  }
339  }
340  catch (Exception const& e) {
341  rc = e.returnCode();
342  printArtException(e, "art");
343  }
344  catch (cet::exception const& e) {
345  rc = 65;
346  printArtException(e, "art");
347  }
348  catch (detail::collected_exception const& e) {
349  rc = 1;
350  // LogSystem already adds a newline, so trim the one that's
351  // already in the exception message.
352  std::string const msg{e.what()};
353  mf::LogSystem("ArtException") << msg.substr(0, msg.find_last_of("\n"));
354  }
355  catch (bad_alloc const& bda) {
356  rc = 68;
357  printBadAllocException("art");
358  }
359  catch (exception const& e) {
360  rc = 66;
361  printStdException(e, "art");
362  }
363  catch (...) {
364  rc = 67;
365  printUnknownException("art");
366  }
367  return rc;
368  }
369 
370 } // namespace art
void SetIteration(string const &val)
static ParameterSetID const & put(ParameterSet const &ps)
decltype(auto) constexpr cend(T &&obj)
ADL-aware version of std::cend.
Definition: StdUtils.h:93
bool exists_outside_prolog(fhicl::intermediate_table const &config, std::string const &key)
void print_config_summary(fhicl::ParameterSet const &pset, std::string const &verbosity, EnabledModules const &enabled_modules)
static ParameterSet make(intermediate_table const &tbl)
Definition: ParameterSet.cc:68
MaybeLogger_< ELseverityLevel::ELsev_info, false > LogInfo
char const * what() const noexcept override
STL namespace.
void printStdException(std::exception const &e, char const *prog)
std::atomic< int > shutdown_flag
void printUnknownException(char const *prog)
EnabledModules prune_config_if_enabled(bool prune_config, bool report_enabled, fhicl::intermediate_table &config)
void StartMessageFacility(fhicl::ParameterSet const &pset, string const &applicationName)
T get(std::string const &key) const
Definition: ParameterSet.h:314
std::enable_if_t< std::is_convertible_v< T, std::string >, std::string > fhicl_key(T const &name)
Definition: fhicl_key.h:12
std::string to_indented_string() const
std::vector< std::unique_ptr< art::OptionsHandler >> OptionsHandlers
MaybeLogger_< ELseverityLevel::ELsev_severe, false > LogSystem
bool has_key(std::string const &key) const
double value
Definition: spectrum.C:18
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
bool output_to_stderr(std::string const &spec)
Definition: output_to.cc:15
debug_processing
Definition: run_art.cc:99
int run_art(int argc, char **argv, bpo::options_description &all_desc, OptionsHandlers &&handlers)
Definition: run_art.cc:142
T get(std::string const &name)
void printArtException(cet::exception const &e, char const *prog)
constexpr int info_success()
Definition: info_success.h:8
decltype(auto) constexpr cbegin(T &&obj)
ADL-aware version of std::cbegin.
Definition: StdUtils.h:85
Definition: MVAAlg.h:12
void printBadAllocException(char const *prog)
Float_t e
Definition: plot.C:35
bool output_to_stdout(std::string const &spec)
Definition: output_to.cc:21
cet::coded_exception< error, detail::translate > exception
Definition: exception.h:33