37 #include "ompl/tools/benchmark/Benchmark.h"
38 #include "ompl/tools/benchmark/MachineSpecs.h"
39 #include "ompl/util/Time.h"
40 #include "ompl/config.h"
41 #include "ompl/util/String.h"
42 #include <boost/scoped_ptr.hpp>
45 #include <condition_variable>
56 static std::string getResultsFilename(
const Benchmark::CompleteExperiment &exp)
58 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".log";
63 static std::string getConsoleFilename(
const Benchmark::CompleteExperiment &exp)
65 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".console";
78 RunPlanner(
const Benchmark *benchmark)
79 : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0)
89 double getTimeUsed()
const
99 base::PlannerStatus getStatus()
const
104 const Benchmark::RunProgressData &getRunProgressData()
const
106 return runProgressData_;
120 return terminationCondition(maxMem, endtime);
130 boost::scoped_ptr<std::thread> t;
131 if (planner->getPlannerProgressProperties().size() > 0)
132 t.reset(
new std::thread([
this, &planner, timeBetweenUpdates]
134 collectProgressProperties(planner->getPlannerProgressProperties(),
137 status_ = planner->solve(ptc, 0.1);
140 solvedCondition_.notify_all();
141 solvedFlag_.unlock();
145 catch (std::runtime_error &e)
147 std::stringstream es;
148 es <<
"There was an error executing planner " << benchmark_->getStatus().activePlanner
149 <<
", run = " << benchmark_->getStatus().activeRun << std::endl;
150 es <<
"*** " << e.what() << std::endl;
151 std::cerr << es.str();
164 std::unique_lock<std::mutex> ulock(solvedFlag_);
167 if (solvedCondition_.wait_for(ulock, timePerUpdate) == std::cv_status::no_timeout)
173 std::map<std::string, std::string> data;
174 data[
"time REAL"] = timeStamp;
175 for (
const auto &property : properties)
177 data[
property.first] =
property.second();
179 runProgressData_.push_back(data);
184 const Benchmark *benchmark_;
187 base::PlannerStatus status_;
188 Benchmark::RunProgressData runProgressData_;
192 std::mutex solvedFlag_;
193 std::condition_variable solvedCondition_;
203 std::ofstream fout(filename);
212 if (getResultsFilename(
exp_) != std::string(filename))
215 OMPL_ERROR(
"Unable to write results to '%s'", filename);
222 std::string filename = getResultsFilename(exp_);
223 return saveResultsToFile(filename.c_str());
228 if (exp_.planners.empty())
230 OMPL_WARN(
"There is no experimental data to save");
240 out <<
"OMPL version " << OMPL_VERSION << std::endl;
241 out <<
"Experiment " << (exp_.name.empty() ?
"NO_NAME" : exp_.name) << std::endl;
243 out << exp_.parameters.size() <<
" experiment properties" << std::endl;
244 for (
const auto ¶meter : exp_.parameters)
245 out << parameter.first <<
" = " << parameter.second << std::endl;
247 out <<
"Running on " << (exp_.host.empty() ?
"UNKNOWN" : exp_.host) << std::endl;
249 out <<
"<<<|" << std::endl << exp_.setupInfo <<
"|>>>" << std::endl;
250 out <<
"<<<|" << std::endl << exp_.cpuInfo <<
"|>>>" << std::endl;
252 out << exp_.seed <<
" is the random seed" << std::endl;
253 out << exp_.maxTime <<
" seconds per run" << std::endl;
254 out << exp_.maxMem <<
" MB per run" << std::endl;
255 out << exp_.runCount <<
" runs per planner" << std::endl;
256 out << exp_.totalDuration <<
" seconds spent to collect the data" << std::endl;
259 out <<
"1 enum type" << std::endl;
265 out << exp_.planners.size() <<
" planners" << std::endl;
267 for (
const auto &planner : exp_.planners)
269 out << planner.name << std::endl;
272 std::vector<std::string> properties;
273 for (
auto &property : planner.common)
274 properties.push_back(property.first);
275 std::sort(properties.begin(), properties.end());
278 out << properties.size() <<
" common properties" << std::endl;
279 for (
auto &property : properties)
281 auto it = planner.common.find(property);
282 out << it->first <<
" = " << it->second << std::endl;
286 std::map<std::string, bool> propSeen;
287 for (
auto &run : planner.runs)
288 for (
auto &property : run)
289 propSeen[
property.first] =
true;
293 for (
auto &it : propSeen)
294 properties.push_back(it.first);
295 std::sort(properties.begin(), properties.end());
298 out << properties.size() <<
" properties for each run" << std::endl;
299 for (
auto &property : properties)
300 out <<
property << std::endl;
303 out << planner.runs.size() <<
" runs" << std::endl;
304 for (
auto &run : planner.runs)
306 for (
auto &property : properties)
308 auto it = run.find(property);
317 if (planner.runsProgressData.size() > 0)
320 out << planner.progressPropertyNames.size() <<
" progress properties for each run" << std::endl;
322 for (
const auto &progPropName : planner.progressPropertyNames)
324 out << progPropName << std::endl;
327 out << planner.runsProgressData.size() <<
" runs" << std::endl;
328 for (
const auto &r : planner.runsProgressData)
331 for (
const auto &t : r)
334 for (
const auto &iter : t)
336 out << iter.second <<
",";
348 out <<
'.' << std::endl;
358 if (!gsetup_->getSpaceInformation()->isSetup())
359 gsetup_->getSpaceInformation()->setup();
363 if (!csetup_->getSpaceInformation()->isSetup())
364 csetup_->getSpaceInformation()->setup();
367 if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
373 if (planners_.empty())
375 OMPL_ERROR(
"There are no planners to benchmark");
379 status_.running =
true;
380 exp_.totalDuration = 0.0;
393 exp_.planners.clear();
394 exp_.planners.resize(planners_.size());
396 const base::ProblemDefinitionPtr &pdef =
397 gsetup_ ? gsetup_->getProblemDefinition() : csetup_->getProblemDefinition();
399 for (
unsigned int i = 0; i < planners_.size(); ++i)
402 planners_[i]->setProblemDefinition(pdef);
403 if (!planners_[i]->isSetup())
404 planners_[i]->setup();
405 exp_.planners[i].name = (gsetup_ ?
"geometric_" :
"control_") + planners_[i]->getName();
406 OMPL_INFORM(
"Configured %s", exp_.planners[i].name.c_str());
410 OMPL_INFORM(
"Saving planner setup information ...");
412 std::stringstream setupInfo;
414 gsetup_->print(setupInfo);
416 csetup_->print(setupInfo);
417 setupInfo << std::endl <<
"Properties of benchmarked planners:" << std::endl;
418 for (
auto &planner : planners_)
419 planner->printProperties(setupInfo);
421 exp_.setupInfo = setupInfo.str();
427 boost::scoped_ptr<msg::OutputHandlerFile> ohf;
437 boost::scoped_ptr<ompl::time::ProgressDisplay> progress;
440 std::cout <<
"Running experiment " << exp_.name <<
"." << std::endl;
442 std::cout <<
"Each planner will be executed " << req.
runCount <<
" times for at most " << req.
maxTime <<
" seconds.";
444 std::cout <<
"Each planner will be executed as many times as possible within " << req.
maxTime <<
" seconds.";
445 std::cout <<
" Memory is limited at " << req.
maxMem <<
"MB." << std::endl;
452 for (
unsigned int i = 0; i < planners_.size(); ++i)
454 status_.activePlanner = exp_.planners[i].name;
460 OMPL_INFORM(
"Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
461 plannerSwitch_(planners_[i]);
462 OMPL_INFORM(
"Completed execution of planner-switch event");
465 catch (std::runtime_error &e)
467 std::stringstream es;
468 es <<
"There was an error executing the planner-switch event for planner " << status_.activePlanner
470 es <<
"*** " << e.what() << std::endl;
471 std::cerr << es.str();
478 planners_[i]->params().getParams(exp_.planners[i].common);
479 planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
482 exp_.planners[i].progressPropertyNames.emplace_back(
"time REAL");
483 for (
const auto &property : planners_[i]->getPlannerProgressProperties())
485 exp_.planners[i].progressPropertyNames.push_back(property.first);
487 std::sort(exp_.planners[i].progressPropertyNames.begin(), exp_.planners[i].progressPropertyNames.end());
494 status_.activeRun = j;
495 status_.progressPercentage = req.
runCount ?
496 (double)(100 * (req.
runCount * i + j)) / (
double)(planners_.size() * req.
runCount) :
497 (
double)(100 * i) / (
double)(planners_.size());
500 while (status_.progressPercentage > progress->count())
503 OMPL_INFORM(
"Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
508 planners_[i]->clear();
511 gsetup_->getProblemDefinition()->clearSolutionPaths();
512 gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
516 csetup_->getProblemDefinition()->clearSolutionPaths();
517 csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
520 catch (std::runtime_error &e)
522 std::stringstream es;
523 es <<
"There was an error while preparing for run " << status_.activeRun <<
" of planner "
524 << status_.activePlanner << std::endl;
525 es <<
"*** " << e.what() << std::endl;
526 std::cerr << es.str();
535 OMPL_INFORM(
"Executing pre-run event for run %d of planner %s ...", status_.activeRun,
536 status_.activePlanner.c_str());
537 preRun_(planners_[i]);
538 OMPL_INFORM(
"Completed execution of pre-run event");
541 catch (std::runtime_error &e)
543 std::stringstream es;
544 es <<
"There was an error executing the pre-run event for run " << status_.activeRun <<
" of planner "
545 << status_.activePlanner << std::endl;
546 es <<
"*** " << e.what() << std::endl;
547 std::cerr << es.str();
553 bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
561 run[
"memory REAL"] =
ompl::toString((
double)rp.getMemUsed() / (1024.0 * 1024.0));
565 run[
"solved BOOLEAN"] = std::to_string(gsetup_->haveExactSolutionPath());
566 run[
"valid segment fraction REAL"] =
567 ompl::toString(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
571 run[
"solved BOOLEAN"] = std::to_string(csetup_->haveExactSolutionPath());
572 run[
"valid segment fraction REAL"] =
573 ompl::toString(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
580 run[
"approximate solution BOOLEAN"] =
581 std::to_string(gsetup_->getProblemDefinition()->hasApproximateSolution());
582 run[
"solution difference REAL"] =
583 ompl::toString(gsetup_->getProblemDefinition()->getSolutionDifference());
584 run[
"solution length REAL"] =
ompl::toString(gsetup_->getSolutionPath().length());
585 run[
"solution smoothness REAL"] =
ompl::toString(gsetup_->getSolutionPath().smoothness());
586 run[
"solution clearance REAL"] =
ompl::toString(gsetup_->getSolutionPath().clearance());
587 run[
"solution segments INTEGER"] =
588 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
589 run[
"correct solution BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
591 unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
592 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
593 run[
"correct solution strict BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
594 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
600 gsetup_->simplifySolution();
603 run[
"simplified solution length REAL"] =
605 run[
"simplified solution smoothness REAL"] =
607 run[
"simplified solution clearance REAL"] =
609 run[
"simplified solution segments INTEGER"] =
610 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
611 run[
"simplified correct solution BOOLEAN"] =
612 std::to_string(gsetup_->getSolutionPath().check());
613 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
614 run[
"simplified correct solution strict BOOLEAN"] =
615 std::to_string(gsetup_->getSolutionPath().check());
616 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
621 run[
"approximate solution BOOLEAN"] =
622 std::to_string(csetup_->getProblemDefinition()->hasApproximateSolution());
623 run[
"solution difference REAL"] =
624 ompl::toString(csetup_->getProblemDefinition()->getSolutionDifference());
625 run[
"solution length REAL"] =
ompl::toString(csetup_->getSolutionPath().length());
626 run[
"solution clearance REAL"] =
627 ompl::toString(csetup_->getSolutionPath().asGeometric().clearance());
628 run[
"solution segments INTEGER"] = std::to_string(csetup_->getSolutionPath().getControlCount());
629 run[
"correct solution BOOLEAN"] = std::to_string(csetup_->getSolutionPath().check());
633 base::PlannerData pd(gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
634 planners_[i]->getPlannerData(pd);
635 run[
"graph states INTEGER"] = std::to_string(pd.
numVertices());
636 run[
"graph motions INTEGER"] = std::to_string(pd.
numEdges());
639 run[prop.first] = prop.second;
646 OMPL_INFORM(
"Executing post-run event for run %d of planner %s ...", status_.activeRun,
647 status_.activePlanner.c_str());
648 postRun_(planners_[i], run);
649 OMPL_INFORM(
"Completed execution of post-run event");
652 catch (std::runtime_error &e)
654 std::stringstream es;
655 es <<
"There was an error in the execution of the post-run event for run " << status_.activeRun
656 <<
" of planner " << status_.activePlanner << std::endl;
657 es <<
"*** " << e.what() << std::endl;
658 std::cerr << es.str();
662 exp_.planners[i].runs.push_back(run);
666 if (planners_[i]->getPlannerProgressProperties().size() > 0)
668 exp_.planners[i].runsProgressData.push_back(rp.getRunProgressData());
671 catch (std::runtime_error &e)
673 std::stringstream es;
674 es <<
"There was an error in the extraction of planner results: planner = " << status_.activePlanner
675 <<
", run = " << status_.activePlanner << std::endl;
676 es <<
"*** " << e.what() << std::endl;
677 std::cerr << es.str();
684 maxTime -= rp.getTimeUsed();
694 planners_[i]->clear();
697 status_.running =
false;
698 status_.progressPercentage = 100.0;
701 while (status_.progressPercentage > progress->count())
703 std::cout << std::endl;