Benchmark.cpp
1 /*********************************************************************
2 * Software License Agreement (BSD License)
3 *
4 * Copyright (c) 2010, Rice University
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 * * Neither the name of the Rice University nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 *********************************************************************/
34 
35 /* Author: Ioan Sucan, Luis G. Torres */
36 
37 #include "ompl/tools/benchmark/Benchmark.h"
38 #include "ompl/tools/benchmark/MachineSpecs.h"
39 #include "ompl/util/Time.h"
40 #include "ompl/config.h"
41 #include <boost/scoped_ptr.hpp>
42 #include <boost/progress.hpp>
43 #include <thread>
44 #include <mutex>
45 #include <condition_variable>
46 #include <fstream>
47 #include <sstream>
48 
50 namespace ompl
51 {
52  namespace tools
53  {
56  static std::string getResultsFilename(const Benchmark::CompleteExperiment &exp)
57  {
58  return "ompl_" + exp.host + "_" + time::as_string(exp.startTime) + ".log";
59  }
60 
63  static std::string getConsoleFilename(const Benchmark::CompleteExperiment &exp)
64  {
65  return "ompl_" + exp.host + "_" + time::as_string(exp.startTime) + ".console";
66  }
67 
68  static bool terminationCondition(const machine::MemUsage_t maxMem, const time::point &endTime)
69  {
70  if (time::now() < endTime && machine::getProcessMemoryUsage() < maxMem)
71  return false;
72  return true;
73  }
74 
75  class RunPlanner
76  {
77  public:
78  RunPlanner(const Benchmark *benchmark, bool useThreads)
79  : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0), useThreads_(useThreads)
80  {
81  }
82 
83  void run(const base::PlannerPtr &planner, const machine::MemUsage_t memStart,
84  const machine::MemUsage_t maxMem, const double maxTime, const double timeBetweenUpdates)
85  {
86  // if (!useThreads_)
87  // {
88  runThread(planner, memStart + maxMem, time::seconds(maxTime), time::seconds(timeBetweenUpdates));
89  // return;
90  // }
91 
92  // std::thread t([planner, memStart, maxMem, maxTime, timeBetweenUpdates]
93  // {
94  // runThread(planner, memStart + maxMem, time::seconds(maxTime),
95  // time::seconds(timeBetweenUpdates)); });
96  // }
97 
98  // allow 25% more time than originally specified, in order to detect planner termination
99  // if (!t.try_join_for(time::seconds(maxTime * 1.25)))
100  // {
101  // status_ = base::PlannerStatus::CRASH;
102  //
103  // std::stringstream es;
104  // es << "Planner " << benchmark_->getStatus().activePlanner << " did not complete run " <<
105  // benchmark_->getStatus().activeRun
106  // << " within the specified amount of time (possible crash). Attempting to force termination of
107  // planning thread ..." << std::endl;
108  // std::cerr << es.str();
109  // OMPL_ERROR(es.str().c_str());
110  //
111  // t.interrupt();
112  // t.join();
113  //
114  // std::string m = "Planning thread cancelled";
115  // std::cerr << m << std::endl;
116  // OMPL_ERROR(m.c_str());
117  // }
118 
119  // if (memStart < memUsed_)
120  // memUsed_ -= memStart;
121  // else
122  // memUsed_ = 0;
123  }
124 
125  double getTimeUsed() const
126  {
127  return timeUsed_;
128  }
129 
130  machine::MemUsage_t getMemUsed() const
131  {
132  return memUsed_;
133  }
134 
135  base::PlannerStatus getStatus() const
136  {
137  return status_;
138  }
139 
140  const Benchmark::RunProgressData &getRunProgressData() const
141  {
142  return runProgressData_;
143  }
144 
145  private:
146  void runThread(const base::PlannerPtr &planner, const machine::MemUsage_t maxMem,
147  const time::duration &maxDuration, const time::duration &timeBetweenUpdates)
148  {
149  time::point timeStart = time::now();
150 
151  try
152  {
153  const time::point endtime = time::now() + maxDuration;
154  base::PlannerTerminationConditionFn ptc([maxMem, endtime]
155  {
156  return terminationCondition(maxMem, endtime);
157  });
158  solved_ = false;
159  // Only launch the planner progress property
160  // collector if there is any data for it to report
161  //
162  // \todo issue here is that at least one sample
163  // always gets taken before planner even starts;
164  // might be worth adding a short wait time before
165  // collector begins sampling
166  boost::scoped_ptr<std::thread> t;
167  if (planner->getPlannerProgressProperties().size() > 0)
168  t.reset(new std::thread([this, &planner, timeBetweenUpdates]
169  {
170  collectProgressProperties(planner->getPlannerProgressProperties(),
171  timeBetweenUpdates);
172  }));
173  status_ = planner->solve(ptc, 0.1);
174  solvedFlag_.lock();
175  solved_ = true;
176  solvedCondition_.notify_all();
177  solvedFlag_.unlock();
178  if (t)
179  t->join(); // maybe look into interrupting even if planner throws an exception
180  }
181  catch (std::runtime_error &e)
182  {
183  std::stringstream es;
184  es << "There was an error executing planner " << benchmark_->getStatus().activePlanner
185  << ", run = " << benchmark_->getStatus().activeRun << std::endl;
186  es << "*** " << e.what() << std::endl;
187  std::cerr << es.str();
188  OMPL_ERROR(es.str().c_str());
189  }
190 
191  timeUsed_ = time::seconds(time::now() - timeStart);
192  memUsed_ = machine::getProcessMemoryUsage();
193  }
194 
195  void collectProgressProperties(const base::Planner::PlannerProgressProperties &properties,
196  const time::duration &timePerUpdate)
197  {
198  time::point timeStart = time::now();
199 
200  std::unique_lock<std::mutex> ulock(solvedFlag_);
201  while (!solved_)
202  {
203  if (solvedCondition_.wait_for(ulock, timePerUpdate) == std::cv_status::no_timeout)
204  return;
205  else
206  {
207  double timeInSeconds = time::seconds(time::now() - timeStart);
208  std::string timeStamp = std::to_string(timeInSeconds);
209  std::map<std::string, std::string> data;
210  data["time REAL"] = timeStamp;
211  for (const auto &property : properties)
212  {
213  data[property.first] = property.second();
214  }
215  runProgressData_.push_back(data);
216  }
217  }
218  }
219 
220  const Benchmark *benchmark_;
221  double timeUsed_;
222  machine::MemUsage_t memUsed_;
223  base::PlannerStatus status_;
224  bool useThreads_;
225  Benchmark::RunProgressData runProgressData_;
226 
227  // variables needed for progress property collection
228  bool solved_;
229  std::mutex solvedFlag_;
230  std::condition_variable solvedCondition_;
231  };
232  }
233 }
235 
236 bool ompl::tools::Benchmark::saveResultsToFile(const char *filename) const
237 {
238  bool result = false;
239 
240  std::ofstream fout(filename);
241  if (fout.good())
242  {
243  result = saveResultsToStream(fout);
244  OMPL_INFORM("Results saved to '%s'", filename);
245  }
246  else
247  {
248  // try to save to a different file, if we can
249  if (getResultsFilename(exp_) != std::string(filename))
250  result = saveResultsToFile();
251 
252  OMPL_ERROR("Unable to write results to '%s'", filename);
253  }
254  return result;
255 }
256 
258 {
259  std::string filename = getResultsFilename(exp_);
260  return saveResultsToFile(filename.c_str());
261 }
262 
263 bool ompl::tools::Benchmark::saveResultsToStream(std::ostream &out) const
264 {
265  if (exp_.planners.empty())
266  {
267  OMPL_WARN("There is no experimental data to save");
268  return false;
269  }
270 
271  if (!out.good())
272  {
273  OMPL_ERROR("Unable to write to stream");
274  return false;
275  }
276 
277  out << "OMPL version " << OMPL_VERSION << std::endl;
278  out << "Experiment " << (exp_.name.empty() ? "NO_NAME" : exp_.name) << std::endl;
279 
280  out << exp_.parameters.size() << " experiment properties" << std::endl;
281  for (const auto &parameter : exp_.parameters)
282  out << parameter.first << " = " << parameter.second << std::endl;
283 
284  out << "Running on " << (exp_.host.empty() ? "UNKNOWN" : exp_.host) << std::endl;
285  out << "Starting at " << time::as_string(exp_.startTime) << std::endl;
286  out << "<<<|" << std::endl
287  << exp_.setupInfo << "|>>>" << std::endl;
288  out << "<<<|" << std::endl
289  << exp_.cpuInfo << "|>>>" << std::endl;
290 
291  out << exp_.seed << " is the random seed" << std::endl;
292  out << exp_.maxTime << " seconds per run" << std::endl;
293  out << exp_.maxMem << " MB per run" << std::endl;
294  out << exp_.runCount << " runs per planner" << std::endl;
295  out << exp_.totalDuration << " seconds spent to collect the data" << std::endl;
296 
297  // change this if more enum types are added
298  out << "1 enum type" << std::endl;
299  out << "status";
300  for (unsigned int i = 0; i < base::PlannerStatus::TYPE_COUNT; ++i)
301  out << '|' << base::PlannerStatus(static_cast<base::PlannerStatus::StatusType>(i)).asString();
302  out << std::endl;
303 
304  out << exp_.planners.size() << " planners" << std::endl;
305 
306  for (const auto &planner : exp_.planners)
307  {
308  out << planner.name << std::endl;
309 
310  // get names of common properties
311  std::vector<std::string> properties;
312  for (auto &property : planner.common)
313  properties.push_back(property.first);
314  std::sort(properties.begin(), properties.end());
315 
316  // print names & values of common properties
317  out << properties.size() << " common properties" << std::endl;
318  for (auto &property : properties)
319  {
320  auto it = planner.common.find(property);
321  out << it->first << " = " << it->second << std::endl;
322  }
323 
324  // construct the list of all possible properties for all runs
325  std::map<std::string, bool> propSeen;
326  for (auto &run : planner.runs)
327  for (auto &property : run)
328  propSeen[property.first] = true;
329 
330  properties.clear();
331 
332  for (auto &it : propSeen)
333  properties.push_back(it.first);
334  std::sort(properties.begin(), properties.end());
335 
336  // print the property names
337  out << properties.size() << " properties for each run" << std::endl;
338  for (auto &propertie : properties)
339  out << propertie << std::endl;
340 
341  // print the data for each run
342  out << planner.runs.size() << " runs" << std::endl;
343  for (auto &run : planner.runs)
344  {
345  for (auto &property : properties)
346  {
347  auto it = run.find(property);
348  if (it != run.end())
349  out << it->second;
350  out << "; ";
351  }
352  out << std::endl;
353  }
354 
355  // print the run progress data if it was reported
356  if (planner.runsProgressData.size() > 0)
357  {
358  // Print number of progress properties
359  out << planner.progressPropertyNames.size() << " progress properties for each run" << std::endl;
360  // Print progress property names
361  for (const auto &progPropName : planner.progressPropertyNames)
362  {
363  out << progPropName << std::endl;
364  }
365  // Print progress properties for each run
366  out << planner.runsProgressData.size() << " runs" << std::endl;
367  for (const auto &r : planner.runsProgressData)
368  {
369  // For each time point
370  for (const auto &t : r)
371  {
372  // Print each of the properties at that time point
373  for (const auto &iter : t)
374  {
375  out << iter.second << ",";
376  }
377 
378  // Separate time points by semicolons
379  out << ";";
380  }
381 
382  // Separate runs by newlines
383  out << std::endl;
384  }
385  }
386 
387  out << '.' << std::endl;
388  }
389  return true;
390 }
391 
393 {
394  // sanity checks
395  if (gsetup_)
396  {
397  if (!gsetup_->getSpaceInformation()->isSetup())
398  gsetup_->getSpaceInformation()->setup();
399  }
400  else
401  {
402  if (!csetup_->getSpaceInformation()->isSetup())
403  csetup_->getSpaceInformation()->setup();
404  }
405 
406  if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
407  {
408  OMPL_ERROR("No goal defined");
409  return;
410  }
411 
412  if (planners_.empty())
413  {
414  OMPL_ERROR("There are no planners to benchmark");
415  return;
416  }
417 
418  status_.running = true;
419  exp_.totalDuration = 0.0;
420  exp_.maxTime = req.maxTime;
421  exp_.maxMem = req.maxMem;
422  exp_.runCount = req.runCount;
423  exp_.host = machine::getHostname();
424  exp_.cpuInfo = machine::getCPUInfo();
425  exp_.seed = RNG::getSeed();
426 
427  exp_.startTime = time::now();
428 
429  OMPL_INFORM("Configuring planners ...");
430 
431  // clear previous experimental data
432  exp_.planners.clear();
433  exp_.planners.resize(planners_.size());
434 
435  const base::ProblemDefinitionPtr &pdef =
436  gsetup_ ? gsetup_->getProblemDefinition() : csetup_->getProblemDefinition();
437  // set up all the planners
438  for (unsigned int i = 0; i < planners_.size(); ++i)
439  {
440  // configure the planner
441  planners_[i]->setProblemDefinition(pdef);
442  if (!planners_[i]->isSetup())
443  planners_[i]->setup();
444  exp_.planners[i].name = (gsetup_ ? "geometric_" : "control_") + planners_[i]->getName();
445  OMPL_INFORM("Configured %s", exp_.planners[i].name.c_str());
446  }
447 
448  OMPL_INFORM("Done configuring planners.");
449  OMPL_INFORM("Saving planner setup information ...");
450 
451  std::stringstream setupInfo;
452  if (gsetup_)
453  gsetup_->print(setupInfo);
454  else
455  csetup_->print(setupInfo);
456  setupInfo << std::endl
457  << "Properties of benchmarked planners:" << std::endl;
458  for (auto &planner : planners_)
459  planner->printProperties(setupInfo);
460 
461  exp_.setupInfo = setupInfo.str();
462 
463  OMPL_INFORM("Done saving information");
464 
465  OMPL_INFORM("Beginning benchmark");
467  boost::scoped_ptr<msg::OutputHandlerFile> ohf;
468  if (req.saveConsoleOutput)
469  {
470  ohf.reset(new msg::OutputHandlerFile(getConsoleFilename(exp_).c_str()));
471  msg::useOutputHandler(ohf.get());
472  }
473  else
475  OMPL_INFORM("Beginning benchmark");
476 
477  boost::scoped_ptr<boost::progress_display> progress;
478  if (req.displayProgress)
479  {
480  std::cout << "Running experiment " << exp_.name << "." << std::endl;
481  std::cout << "Each planner will be executed " << req.runCount << " times for at most " << req.maxTime
482  << " seconds. Memory is limited at " << req.maxMem << "MB." << std::endl;
483  progress.reset(new boost::progress_display(100, std::cout));
484  }
485 
487  machine::MemUsage_t maxMemBytes = (machine::MemUsage_t)(req.maxMem * 1024 * 1024);
488 
489  for (unsigned int i = 0; i < planners_.size(); ++i)
490  {
491  status_.activePlanner = exp_.planners[i].name;
492  // execute planner switch event, if set
493  try
494  {
495  if (plannerSwitch_)
496  {
497  OMPL_INFORM("Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
498  plannerSwitch_(planners_[i]);
499  OMPL_INFORM("Completed execution of planner-switch event");
500  }
501  }
502  catch (std::runtime_error &e)
503  {
504  std::stringstream es;
505  es << "There was an error executing the planner-switch event for planner " << status_.activePlanner
506  << std::endl;
507  es << "*** " << e.what() << std::endl;
508  std::cerr << es.str();
509  OMPL_ERROR(es.str().c_str());
510  }
511  if (gsetup_)
512  gsetup_->setup();
513  else
514  csetup_->setup();
515  planners_[i]->params().getParams(exp_.planners[i].common);
516  planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
517 
518  // Add planner progress property names to struct
519  exp_.planners[i].progressPropertyNames.push_back("time REAL");
520  base::Planner::PlannerProgressProperties::const_iterator iter;
521  for (iter = planners_[i]->getPlannerProgressProperties().begin();
522  iter != planners_[i]->getPlannerProgressProperties().end(); ++iter)
523  {
524  exp_.planners[i].progressPropertyNames.push_back(iter->first);
525  }
526  std::sort(exp_.planners[i].progressPropertyNames.begin(), exp_.planners[i].progressPropertyNames.end());
527 
528  // run the planner
529  for (unsigned int j = 0; j < req.runCount; ++j)
530  {
531  status_.activeRun = j;
532  status_.progressPercentage =
533  (double)(100 * (req.runCount * i + j)) / (double)(planners_.size() * req.runCount);
534 
535  if (req.displayProgress)
536  while (status_.progressPercentage > progress->count())
537  ++(*progress);
538 
539  OMPL_INFORM("Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
540 
541  // make sure all planning data structures are cleared
542  try
543  {
544  planners_[i]->clear();
545  if (gsetup_)
546  {
547  gsetup_->getProblemDefinition()->clearSolutionPaths();
548  gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
549  }
550  else
551  {
552  csetup_->getProblemDefinition()->clearSolutionPaths();
553  csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
554  }
555  }
556  catch (std::runtime_error &e)
557  {
558  std::stringstream es;
559  es << "There was an error while preparing for run " << status_.activeRun << " of planner "
560  << status_.activePlanner << std::endl;
561  es << "*** " << e.what() << std::endl;
562  std::cerr << es.str();
563  OMPL_ERROR(es.str().c_str());
564  }
565 
566  // execute pre-run event, if set
567  try
568  {
569  if (preRun_)
570  {
571  OMPL_INFORM("Executing pre-run event for run %d of planner %s ...", status_.activeRun,
572  status_.activePlanner.c_str());
573  preRun_(planners_[i]);
574  OMPL_INFORM("Completed execution of pre-run event");
575  }
576  }
577  catch (std::runtime_error &e)
578  {
579  std::stringstream es;
580  es << "There was an error executing the pre-run event for run " << status_.activeRun << " of planner "
581  << status_.activePlanner << std::endl;
582  es << "*** " << e.what() << std::endl;
583  std::cerr << es.str();
584  OMPL_ERROR(es.str().c_str());
585  }
586 
587  RunPlanner rp(this, req.useThreads);
588  rp.run(planners_[i], memStart, maxMemBytes, req.maxTime, req.timeBetweenUpdates);
589  bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
590 
591  // store results
592  try
593  {
594  RunProperties run;
595 
596  run["time REAL"] = std::to_string(rp.getTimeUsed());
597  run["memory REAL"] = std::to_string((double)rp.getMemUsed() / (1024.0 * 1024.0));
598  run["status ENUM"] = std::to_string((int)static_cast<base::PlannerStatus::StatusType>(rp.getStatus()));
599  if (gsetup_)
600  {
601  run["solved BOOLEAN"] = std::to_string(gsetup_->haveExactSolutionPath());
602  run["valid segment fraction REAL"] =
603  std::to_string(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
604  }
605  else
606  {
607  run["solved BOOLEAN"] = std::to_string(csetup_->haveExactSolutionPath());
608  run["valid segment fraction REAL"] =
609  std::to_string(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
610  }
611 
612  if (solved)
613  {
614  if (gsetup_)
615  {
616  run["approximate solution BOOLEAN"] =
617  std::to_string(gsetup_->getProblemDefinition()->hasApproximateSolution());
618  run["solution difference REAL"] =
619  std::to_string(gsetup_->getProblemDefinition()->getSolutionDifference());
620  run["solution length REAL"] = std::to_string(gsetup_->getSolutionPath().length());
621  run["solution smoothness REAL"] = std::to_string(gsetup_->getSolutionPath().smoothness());
622  run["solution clearance REAL"] = std::to_string(gsetup_->getSolutionPath().clearance());
623  run["solution segments INTEGER"] =
624  std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
625  run["correct solution BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
626 
627  unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
628  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
629  run["correct solution strict BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
630  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
631 
632  if (req.simplify)
633  {
634  // simplify solution
635  time::point timeStart = time::now();
636  gsetup_->simplifySolution();
637  double timeUsed = time::seconds(time::now() - timeStart);
638  run["simplification time REAL"] = std::to_string(timeUsed);
639  run["simplified solution length REAL"] =
640  std::to_string(gsetup_->getSolutionPath().length());
641  run["simplified solution smoothness REAL"] =
642  std::to_string(gsetup_->getSolutionPath().smoothness());
643  run["simplified solution clearance REAL"] =
644  std::to_string(gsetup_->getSolutionPath().clearance());
645  run["simplified solution segments INTEGER"] =
646  std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
647  run["simplified correct solution BOOLEAN"] =
648  std::to_string(gsetup_->getSolutionPath().check());
649  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
650  run["simplified correct solution strict BOOLEAN"] =
651  std::to_string(gsetup_->getSolutionPath().check());
652  gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
653  }
654  }
655  else
656  {
657  run["approximate solution BOOLEAN"] =
658  std::to_string(csetup_->getProblemDefinition()->hasApproximateSolution());
659  run["solution difference REAL"] =
660  std::to_string(csetup_->getProblemDefinition()->getSolutionDifference());
661  run["solution length REAL"] = std::to_string(csetup_->getSolutionPath().length());
662  run["solution clearance REAL"] =
663  std::to_string(csetup_->getSolutionPath().asGeometric().clearance());
664  run["solution segments INTEGER"] = std::to_string(csetup_->getSolutionPath().getControlCount());
665  run["correct solution BOOLEAN"] = std::to_string(csetup_->getSolutionPath().check());
666  }
667  }
668 
669  base::PlannerData pd(gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
670  planners_[i]->getPlannerData(pd);
671  run["graph states INTEGER"] = std::to_string(pd.numVertices());
672  run["graph motions INTEGER"] = std::to_string(pd.numEdges());
673 
674  for (std::map<std::string, std::string>::const_iterator it = pd.properties.begin();
675  it != pd.properties.end(); ++it)
676  run[it->first] = it->second;
677 
678  // execute post-run event, if set
679  try
680  {
681  if (postRun_)
682  {
683  OMPL_INFORM("Executing post-run event for run %d of planner %s ...", status_.activeRun,
684  status_.activePlanner.c_str());
685  postRun_(planners_[i], run);
686  OMPL_INFORM("Completed execution of post-run event");
687  }
688  }
689  catch (std::runtime_error &e)
690  {
691  std::stringstream es;
692  es << "There was an error in the execution of the post-run event for run " << status_.activeRun
693  << " of planner " << status_.activePlanner << std::endl;
694  es << "*** " << e.what() << std::endl;
695  std::cerr << es.str();
696  OMPL_ERROR(es.str().c_str());
697  }
698 
699  exp_.planners[i].runs.push_back(run);
700 
701  // Add planner progress data from the planner progress
702  // collector if there was anything to report
703  if (planners_[i]->getPlannerProgressProperties().size() > 0)
704  {
705  exp_.planners[i].runsProgressData.push_back(rp.getRunProgressData());
706  }
707  }
708  catch (std::runtime_error &e)
709  {
710  std::stringstream es;
711  es << "There was an error in the extraction of planner results: planner = " << status_.activePlanner
712  << ", run = " << status_.activePlanner << std::endl;
713  es << "*** " << e.what() << std::endl;
714  std::cerr << es.str();
715  OMPL_ERROR(es.str().c_str());
716  }
717  }
718  }
719 
720  status_.running = false;
721  status_.progressPercentage = 100.0;
722  if (req.displayProgress)
723  {
724  while (status_.progressPercentage > progress->count())
725  ++(*progress);
726  std::cout << std::endl;
727  }
728 
729  exp_.totalDuration = time::seconds(time::now() - exp_.startTime);
730 
731  OMPL_INFORM("Benchmark complete");
733  OMPL_INFORM("Benchmark complete");
734 }
Object containing planner generated vertex and edge data. It is assumed that all vertices are unique...
Definition: PlannerData.h:174
std::string getCPUInfo()
Get information about the CPU of the machine in use.
A shared pointer wrapper for ompl::base::ProblemDefinition.
virtual bool saveResultsToStream(std::ostream &out=std::cout) const
Save the results of the benchmark to a stream.
Definition: Benchmark.cpp:263
double maxMem
the maximum amount of memory a planner is allowed to use (MB); 4096.0 by default
Definition: Benchmark.h:177
double timeBetweenUpdates
When collecting time-varying data from a planner during its execution, the planner&#39;s progress will be...
Definition: Benchmark.h:184
std::map< std::string, std::string > RunProperties
The data collected from a run of a planner is stored as key-value pairs.
Definition: Benchmark.h:79
std::function< bool()> PlannerTerminationConditionFn
Signature for functions that decide whether termination conditions have been met for a planner...
void noOutputHandler()
This function instructs ompl that no messages should be outputted. Equivalent to useOutputHandler(nul...
Definition: Console.cpp:95
std::string as_string(const point &p)
Return string representation of point in time.
Definition: Time.h:90
bool displayProgress
flag indicating whether progress is to be displayed or not; true by default
Definition: Benchmark.h:187
unsigned int runCount
the number of times to run each planner; 100 by default
Definition: Benchmark.h:180
bool saveConsoleOutput
flag indicating whether console output is saved (in an automatically generated filename); true by def...
Definition: Benchmark.h:191
duration seconds(double sec)
Return the time duration representing a given number of seconds.
Definition: Time.h:76
bool useThreads
flag indicating whether planner runs should be run in a separate thread. It is advisable to set this ...
Definition: Benchmark.h:196
Main namespace. Contains everything in this library.
Definition: AppBase.h:21
The number of possible status values.
Definition: PlannerStatus.h:72
#define OMPL_ERROR(fmt,...)
Log a formatted error string.
Definition: Console.h:64
double maxTime
the maximum amount of time a planner is allowed to run (seconds); 5.0 by default
Definition: Benchmark.h:174
std::chrono::system_clock::duration duration
Representation of a time duration.
Definition: Time.h:67
A class to store the exit status of Planner::solve()
Definition: PlannerStatus.h:48
Generic class to handle output from a piece of code.
Definition: Console.h:102
static std::uint_fast32_t getSeed()
Get the seed used to generate the seeds of each RNG instance. Passing the returned value to setSeed()...
#define OMPL_WARN(fmt,...)
Log a formatted warning string.
Definition: Console.h:66
MemUsage_t getProcessMemoryUsage()
Get the amount of memory the current process is using. This should work on major platforms (Windows...
Representation of a benchmark request.
Definition: Benchmark.h:156
bool simplify
flag indicating whether simplification should be applied to path; true by default ...
Definition: Benchmark.h:199
point now()
Get the current time point.
Definition: Time.h:70
bool saveResultsToFile() const
Save the results of the benchmark to a file. The name of the file is the current date and time...
Definition: Benchmark.cpp:257
OutputHandler * getOutputHandler()
Get the instance of the OutputHandler currently used. This is nullptr in case there is no output hand...
Definition: Console.cpp:115
Implementation of OutputHandler that saves messages in a file.
Definition: Console.h:127
unsigned long long MemUsage_t
Amount of memory used, in bytes.
Definition: MachineSpecs.h:48
std::string getHostname()
Get the hostname of the machine in use.
std::chrono::system_clock::time_point point
Representation of a point in time.
Definition: Time.h:64
std::string asString() const
Return a string representation.
void useOutputHandler(OutputHandler *oh)
Specify the instance of the OutputHandler to use. By default, this is OutputHandlerSTD.
Definition: Console.cpp:108
std::map< std::string, PlannerProgressProperty > PlannerProgressProperties
A dictionary which maps the name of a progress property to the function to be used for querying that ...
Definition: Planner.h:358
virtual void benchmark(const Request &req)
Benchmark the added planners on the defined problem. Repeated calls clear previously gathered data...
Definition: Benchmark.cpp:392
#define OMPL_INFORM(fmt,...)
Log a formatted information string.
Definition: Console.h:68