37 #include "ompl/tools/benchmark/Benchmark.h"
38 #include "ompl/tools/benchmark/MachineSpecs.h"
39 #include "ompl/util/Time.h"
40 #include "ompl/config.h"
41 #include "ompl/util/String.h"
42 #include <boost/scoped_ptr.hpp>
45 #include <condition_variable>
56 static std::string getResultsFilename(
const Benchmark::CompleteExperiment &exp)
58 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".log";
63 static std::string getConsoleFilename(
const Benchmark::CompleteExperiment &exp)
65 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".console";
78 RunPlanner(
const Benchmark *benchmark)
79 : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0)
89 double getTimeUsed()
const
99 base::PlannerStatus getStatus()
const
104 const Benchmark::RunProgressData &getRunProgressData()
const
106 return runProgressData_;
120 return terminationCondition(maxMem, endtime);
130 boost::scoped_ptr<std::thread> t;
131 if (planner->getPlannerProgressProperties().size() > 0)
132 t.reset(
new std::thread([
this, &planner, timeBetweenUpdates]
134 collectProgressProperties(planner->getPlannerProgressProperties(),
137 status_ = planner->solve(ptc, 0.1);
140 solvedCondition_.notify_all();
141 solvedFlag_.unlock();
145 catch (std::runtime_error &e)
147 std::stringstream es;
148 es <<
"There was an error executing planner " << benchmark_->getStatus().activePlanner
149 <<
", run = " << benchmark_->getStatus().activeRun << std::endl;
150 es <<
"*** " << e.what() << std::endl;
151 std::cerr << es.str();
164 std::unique_lock<std::mutex> ulock(solvedFlag_);
167 if (solvedCondition_.wait_for(ulock, timePerUpdate) == std::cv_status::no_timeout)
173 std::map<std::string, std::string> data;
174 data[
"time REAL"] = timeStamp;
175 for (
const auto &property : properties)
177 data[
property.first] =
property.second();
179 runProgressData_.push_back(data);
184 const Benchmark *benchmark_;
187 base::PlannerStatus status_;
188 Benchmark::RunProgressData runProgressData_;
192 std::mutex solvedFlag_;
193 std::condition_variable solvedCondition_;
203 std::ofstream fout(filename);
212 if (getResultsFilename(
exp_) != std::string(filename))
215 OMPL_ERROR(
"Unable to write results to '%s'", filename);
222 std::string filename = getResultsFilename(exp_);
223 return saveResultsToFile(filename.c_str());
228 if (exp_.planners.empty())
230 OMPL_WARN(
"There is no experimental data to save");
240 out <<
"OMPL version " << OMPL_VERSION << std::endl;
241 out <<
"Experiment " << (exp_.name.empty() ?
"NO_NAME" : exp_.name) << std::endl;
243 out << exp_.parameters.size() <<
" experiment properties" << std::endl;
244 for (
const auto ¶meter : exp_.parameters)
245 out << parameter.first <<
" = " << parameter.second << std::endl;
247 out <<
"Running on " << (exp_.host.empty() ?
"UNKNOWN" : exp_.host) << std::endl;
249 out <<
"<<<|" << std::endl << exp_.setupInfo <<
"|>>>" << std::endl;
250 out <<
"<<<|" << std::endl << exp_.cpuInfo <<
"|>>>" << std::endl;
252 out << exp_.seed <<
" is the random seed" << std::endl;
253 out << exp_.maxTime <<
" seconds per run" << std::endl;
254 out << exp_.maxMem <<
" MB per run" << std::endl;
255 out << exp_.runCount <<
" runs per planner" << std::endl;
256 out << exp_.totalDuration <<
" seconds spent to collect the data" << std::endl;
259 out <<
"1 enum type" << std::endl;
265 out << exp_.planners.size() <<
" planners" << std::endl;
267 for (
const auto &planner : exp_.planners)
269 out << planner.name << std::endl;
272 std::vector<std::string> properties;
273 for (
auto &property : planner.common)
274 properties.push_back(property.first);
275 std::sort(properties.begin(), properties.end());
278 out << properties.size() <<
" common properties" << std::endl;
279 for (
auto &property : properties)
281 auto it = planner.common.find(property);
282 out << it->first <<
" = " << it->second << std::endl;
286 std::map<std::string, bool> propSeen;
287 for (
auto &run : planner.runs)
288 for (
auto &property : run)
289 propSeen[
property.first] =
true;
293 for (
auto &it : propSeen)
294 properties.push_back(it.first);
295 std::sort(properties.begin(), properties.end());
298 out << properties.size() <<
" properties for each run" << std::endl;
299 for (
auto &property : properties)
300 out <<
property << std::endl;
303 out << planner.runs.size() <<
" runs" << std::endl;
304 for (
auto &run : planner.runs)
306 for (
auto &property : properties)
308 auto it = run.find(property);
317 if (planner.runsProgressData.size() > 0)
320 out << planner.progressPropertyNames.size() <<
" progress properties for each run" << std::endl;
322 for (
const auto &progPropName : planner.progressPropertyNames)
324 out << progPropName << std::endl;
327 out << planner.runsProgressData.size() <<
" runs" << std::endl;
328 for (
const auto &r : planner.runsProgressData)
331 for (
const auto &t : r)
334 for (
const auto &iter : t)
336 out << iter.second <<
",";
348 out <<
'.' << std::endl;
358 if (!gsetup_->getSpaceInformation()->isSetup())
359 gsetup_->getSpaceInformation()->setup();
363 if (!csetup_->getSpaceInformation()->isSetup())
364 csetup_->getSpaceInformation()->setup();
367 if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
373 if (planners_.empty())
375 OMPL_ERROR(
"There are no planners to benchmark");
379 status_.running =
true;
380 exp_.totalDuration = 0.0;
393 exp_.planners.clear();
394 exp_.planners.resize(planners_.size());
396 const base::ProblemDefinitionPtr &pdef =
397 gsetup_ ? gsetup_->getProblemDefinition() : csetup_->getProblemDefinition();
399 for (
unsigned int i = 0; i < planners_.size(); ++i)
402 planners_[i]->setProblemDefinition(pdef);
403 if (!planners_[i]->isSetup())
404 planners_[i]->setup();
405 exp_.planners[i].name = (gsetup_ ?
"geometric_" :
"control_") + planners_[i]->getName();
406 OMPL_INFORM(
"Configured %s", exp_.planners[i].name.c_str());
410 OMPL_INFORM(
"Saving planner setup information ...");
412 std::stringstream setupInfo;
414 gsetup_->print(setupInfo);
416 csetup_->print(setupInfo);
417 setupInfo << std::endl <<
"Properties of benchmarked planners:" << std::endl;
418 for (
auto &planner : planners_)
419 planner->printProperties(setupInfo);
421 exp_.setupInfo = setupInfo.str();
427 boost::scoped_ptr<msg::OutputHandlerFile> ohf;
437 boost::scoped_ptr<ompl::time::ProgressDisplay> progress;
440 std::cout <<
"Running experiment " << exp_.name <<
"." << std::endl;
442 std::cout <<
"Each planner will be executed " << req.
runCount <<
" times for at most " << req.
maxTime <<
" seconds.";
444 std::cout <<
"Each planner will be executed as many times as possible within " << req.
maxTime <<
" seconds.";
445 std::cout <<
" Memory is limited at " << req.
maxMem <<
"MB." << std::endl;
452 for (
unsigned int i = 0; i < planners_.size(); ++i)
454 status_.activePlanner = exp_.planners[i].name;
460 OMPL_INFORM(
"Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
461 plannerSwitch_(planners_[i]);
462 OMPL_INFORM(
"Completed execution of planner-switch event");
465 catch (std::runtime_error &e)
467 std::stringstream es;
468 es <<
"There was an error executing the planner-switch event for planner " << status_.activePlanner
470 es <<
"*** " << e.what() << std::endl;
471 std::cerr << es.str();
478 planners_[i]->params().getParams(exp_.planners[i].common);
479 planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
482 exp_.planners[i].progressPropertyNames.emplace_back(
"time REAL");
483 for (
const auto &property : planners_[i]->getPlannerProgressProperties())
485 exp_.planners[i].progressPropertyNames.push_back(property.first);
487 std::sort(exp_.planners[i].progressPropertyNames.begin(), exp_.planners[i].progressPropertyNames.end());
494 status_.activeRun = j;
495 status_.progressPercentage = req.
runCount ?
496 (double)(100 * (req.
runCount * i + j)) / (
double)(planners_.size() * req.
runCount) :
497 (
double)(100 * i) / (
double)(planners_.size());
500 while (status_.progressPercentage > progress->count())
503 OMPL_INFORM(
"Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
508 planners_[i]->clear();
511 gsetup_->getProblemDefinition()->clearSolutionPaths();
512 gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
516 csetup_->getProblemDefinition()->clearSolutionPaths();
517 csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
520 catch (std::runtime_error &e)
522 std::stringstream es;
523 es <<
"There was an error while preparing for run " << status_.activeRun <<
" of planner "
524 << status_.activePlanner << std::endl;
525 es <<
"*** " << e.what() << std::endl;
526 std::cerr << es.str();
535 OMPL_INFORM(
"Executing pre-run event for run %d of planner %s ...", status_.activeRun,
536 status_.activePlanner.c_str());
537 preRun_(planners_[i]);
538 OMPL_INFORM(
"Completed execution of pre-run event");
541 catch (std::runtime_error &e)
543 std::stringstream es;
544 es <<
"There was an error executing the pre-run event for run " << status_.activeRun <<
" of planner "
545 << status_.activePlanner << std::endl;
546 es <<
"*** " << e.what() << std::endl;
547 std::cerr << es.str();
553 bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
561 run[
"memory REAL"] =
ompl::toString((
double)rp.getMemUsed() / (1024.0 * 1024.0));
565 run[
"solved BOOLEAN"] = std::to_string(gsetup_->haveExactSolutionPath());
566 run[
"valid segment fraction REAL"] =
567 ompl::toString(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
571 run[
"solved BOOLEAN"] = std::to_string(csetup_->haveExactSolutionPath());
572 run[
"valid segment fraction REAL"] =
573 ompl::toString(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
580 run[
"approximate solution BOOLEAN"] =
581 std::to_string(gsetup_->getProblemDefinition()->hasApproximateSolution());
582 run[
"solution difference REAL"] =
583 ompl::toString(gsetup_->getProblemDefinition()->getSolutionDifference());
584 run[
"solution length REAL"] =
ompl::toString(gsetup_->getSolutionPath().length());
585 run[
"solution smoothness REAL"] =
ompl::toString(gsetup_->getSolutionPath().smoothness());
586 run[
"solution clearance REAL"] =
ompl::toString(gsetup_->getSolutionPath().clearance());
587 run[
"solution segments INTEGER"] =
588 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
589 run[
"correct solution BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
591 unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
592 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
593 run[
"correct solution strict BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
594 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
600 gsetup_->simplifySolution();
603 run[
"simplified solution length REAL"] =
605 run[
"simplified solution smoothness REAL"] =
607 run[
"simplified solution clearance REAL"] =
609 run[
"simplified solution segments INTEGER"] =
610 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
611 run[
"simplified correct solution BOOLEAN"] =
612 std::to_string(gsetup_->getSolutionPath().check());
613 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
614 run[
"simplified correct solution strict BOOLEAN"] =
615 std::to_string(gsetup_->getSolutionPath().check());
616 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
621 run[
"approximate solution BOOLEAN"] =
622 std::to_string(csetup_->getProblemDefinition()->hasApproximateSolution());
623 run[
"solution difference REAL"] =
624 ompl::toString(csetup_->getProblemDefinition()->getSolutionDifference());
625 run[
"solution length REAL"] =
ompl::toString(csetup_->getSolutionPath().length());
626 run[
"solution clearance REAL"] =
627 ompl::toString(csetup_->getSolutionPath().asGeometric().clearance());
628 run[
"solution segments INTEGER"] = std::to_string(csetup_->getSolutionPath().getControlCount());
629 run[
"correct solution BOOLEAN"] = std::to_string(csetup_->getSolutionPath().check());
633 base::PlannerData pd(gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
634 planners_[i]->getPlannerData(pd);
635 run[
"graph states INTEGER"] = std::to_string(pd.
numVertices());
636 run[
"graph motions INTEGER"] = std::to_string(pd.
numEdges());
639 run[prop.first] = prop.second;
646 OMPL_INFORM(
"Executing post-run event for run %d of planner %s ...", status_.activeRun,
647 status_.activePlanner.c_str());
648 postRun_(planners_[i], run);
649 OMPL_INFORM(
"Completed execution of post-run event");
652 catch (std::runtime_error &e)
654 std::stringstream es;
655 es <<
"There was an error in the execution of the post-run event for run " << status_.activeRun
656 <<
" of planner " << status_.activePlanner << std::endl;
657 es <<
"*** " << e.what() << std::endl;
658 std::cerr << es.str();
662 exp_.planners[i].runs.push_back(run);
666 if (planners_[i]->getPlannerProgressProperties().size() > 0)
668 exp_.planners[i].runsProgressData.push_back(rp.getRunProgressData());
671 catch (std::runtime_error &e)
673 std::stringstream es;
674 es <<
"There was an error in the extraction of planner results: planner = " << status_.activePlanner
675 <<
", run = " << status_.activePlanner << std::endl;
676 es <<
"*** " << e.what() << std::endl;
677 std::cerr << es.str();
684 maxTime -= rp.getTimeUsed();
696 status_.running =
false;
697 status_.progressPercentage = 100.0;
700 while (status_.progressPercentage > progress->count())
702 std::cout << std::endl;
static std::uint_fast32_t getSeed()
Get the seed used to generate the seeds of each RNG instance. Passing the returned value to setSeed()...
Object containing planner generated vertex and edge data. It is assumed that all vertices are unique,...
unsigned int numEdges() const
Retrieve the number of edges in this structure.
unsigned int numVertices() const
Retrieve the number of vertices in this structure.
std::map< std::string, std::string > properties
Any extra properties (key-value pairs) the planner can set.
std::map< std::string, PlannerProgressProperty > PlannerProgressProperties
A dictionary which maps the name of a progress property to the function to be used for querying that ...
Implementation of OutputHandler that saves messages in a file.
Generic class to handle output from a piece of code.
#define OMPL_INFORM(fmt,...)
Log a formatted information string.
#define OMPL_ERROR(fmt,...)
Log a formatted error string.
#define OMPL_WARN(fmt,...)
Log a formatted warning string.
std::function< bool()> PlannerTerminationConditionFn
Signature for functions that decide whether termination conditions have been met for a planner,...
MemUsage_t getProcessMemoryUsage()
Get the amount of memory the current process is using. This should work on major platforms (Windows,...
std::string getHostname()
Get the hostname of the machine in use.
std::string getCPUInfo()
Get information about the CPU of the machine in use.
unsigned long long MemUsage_t
Amount of memory used, in bytes.
OutputHandler * getOutputHandler()
Get the instance of the OutputHandler currently used. This is nullptr in case there is no output hand...
void useOutputHandler(OutputHandler *oh)
Specify the instance of the OutputHandler to use. By default, this is OutputHandlerSTD.
void noOutputHandler()
This function instructs ompl that no messages should be outputted. Equivalent to useOutputHandler(nul...
std::chrono::system_clock::time_point point
Representation of a point in time.
point now()
Get the current time point.
duration seconds(double sec)
Return the time duration representing a given number of seconds.
std::chrono::system_clock::duration duration
Representation of a time duration.
std::string as_string(const point &p)
Return string representation of point in time.
Main namespace. Contains everything in this library.
std::string toString(float val)
convert float to string using classic "C" locale semantics
A class to store the exit status of Planner::solve()
StatusType
The possible values of the status returned by a planner.
@ TYPE_COUNT
The number of possible status values.
std::string asString() const
Return a string representation.