Benchmark.h
1 /*********************************************************************
2 * Software License Agreement (BSD License)
3 *
4 * Copyright (c) 2010, Rice University
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 * * Neither the name of the Rice University nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 *********************************************************************/
34 
35 /* Author: Ioan Sucan */
36 
37 #ifndef OMPL_TOOLS_BENCHMARK_BENCHMARK_
38 #define OMPL_TOOLS_BENCHMARK_BENCHMARK_
39 
40 #include "ompl/geometric/SimpleSetup.h"
41 #include "ompl/control/SimpleSetup.h"
42 
43 namespace ompl
44 {
45  namespace tools
46  {
48  class Benchmark
49  {
50  public:
55  struct Status
56  {
57  Status()
58  {
59  running = false;
60  activeRun = 0;
61  progressPercentage = 0.0;
62  }
63 
65  bool running;
66 
68  std::string activePlanner;
69 
71  unsigned int activeRun;
72 
75  };
76 
79  typedef std::map<std::string, std::string> RunProperties;
80 
81  typedef std::vector<std::map<std::string, std::string>> RunProgressData;
82 
84  typedef std::function<void(const base::PlannerPtr &)> PreSetupEvent;
85 
87  typedef std::function<void(const base::PlannerPtr &, RunProperties &)> PostSetupEvent;
88 
91  {
93  std::string name;
94 
96  std::vector<RunProperties> runs;
97 
100  std::vector<std::string> progressPropertyNames;
101 
104  std::vector<RunProgressData> runsProgressData;
105 
107  RunProperties common;
108 
109  bool operator==(const PlannerExperiment &p) const
110  {
111  return name == p.name && runs == p.runs && common == p.common;
112  }
113  };
114 
117  {
119  std::string name;
120 
122  std::vector<PlannerExperiment> planners;
123 
125  double maxTime;
126 
128  double maxMem;
129 
131  unsigned int runCount;
132 
135 
138 
140  std::string setupInfo;
141 
143  boost::uint32_t seed;
144 
146  std::string host;
147 
149  std::string cpuInfo;
150 
152  std::map<std::string, std::string> parameters;
153  };
154 
156  struct Request
157  {
159  Request(double maxTime = 5.0, double maxMem = 4096.0, unsigned int runCount = 100,
160  double timeBetweenUpdates = 0.05, bool displayProgress = true, bool saveConsoleOutput = true,
161  bool useThreads = true, bool simplify = true)
162  : maxTime(maxTime)
163  , maxMem(maxMem)
164  , runCount(runCount)
165  , timeBetweenUpdates(timeBetweenUpdates)
166  , displayProgress(displayProgress)
167  , saveConsoleOutput(saveConsoleOutput)
168  , useThreads(useThreads)
169  , simplify(simplify)
170  {
171  }
172 
174  double maxTime;
175 
177  double maxMem;
178 
180  unsigned int runCount;
181 
185 
188 
192 
197 
199  bool simplify;
200  };
201 
204  Benchmark(geometric::SimpleSetup &setup, const std::string &name = std::string())
205  : gsetup_(&setup), csetup_(nullptr)
206  {
207  exp_.name = name;
208  }
209 
212  Benchmark(control::SimpleSetup &setup, const std::string &name = std::string())
213  : gsetup_(nullptr), csetup_(&setup)
214  {
215  exp_.name = name;
216  }
217 
218  virtual ~Benchmark() = default;
219 
223  void addExperimentParameter(const std::string &name, const std::string &type, const std::string &value)
224  {
225  exp_.parameters[name + " " + type] = value;
226  }
227 
229  const std::map<std::string, std::string> &getExperimentParameters() const
230  {
231  return exp_.parameters;
232  }
233 
235  std::size_t numExperimentParameters() const
236  {
237  return exp_.parameters.size();
238  }
239 
241  void setExperimentName(const std::string &name)
242  {
243  exp_.name = name;
244  }
245 
247  const std::string &getExperimentName() const
248  {
249  return exp_.name;
250  }
251 
253  void addPlanner(const base::PlannerPtr &planner)
254  {
255  if (planner &&
256  planner->getSpaceInformation().get() !=
257  (gsetup_ != nullptr ? gsetup_->getSpaceInformation().get() : csetup_->getSpaceInformation().get()))
258  throw Exception("Planner instance does not match space information");
259  planners_.push_back(planner);
260  }
261 
264  {
265  planners_.push_back(pa(gsetup_ != nullptr ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation()));
266  }
267 
270  {
271  planners_.clear();
272  }
273 
275  void setPlannerSwitchEvent(const PreSetupEvent &event)
276  {
277  plannerSwitch_ = event;
278  }
279 
281  void setPreRunEvent(const PreSetupEvent &event)
282  {
283  preRun_ = event;
284  }
285 
287  void setPostRunEvent(const PostSetupEvent &event)
288  {
289  postRun_ = event;
290  }
291 
304  virtual void benchmark(const Request &req);
305 
308  const Status &getStatus() const
309  {
310  return status_;
311  }
312 
318  {
319  return exp_;
320  }
321 
323  virtual bool saveResultsToStream(std::ostream &out = std::cout) const;
324 
326  bool saveResultsToFile(const char *filename) const;
327 
330  bool saveResultsToFile() const;
331 
332  protected:
335 
338 
340  std::vector<base::PlannerPtr> planners_;
341 
344 
347 
349  PreSetupEvent plannerSwitch_;
350 
352  PreSetupEvent preRun_;
353 
355  PostSetupEvent postRun_;
356  };
357  }
358 }
359 #endif
const std::string & getExperimentName() const
Get the name of the experiment.
Definition: Benchmark.h:247
bool running
Flag indicating whether benchmarking is running.
Definition: Benchmark.h:65
double maxTime
The maximum allowed time for planner computation during the experiment (seconds)
Definition: Benchmark.h:125
control::SimpleSetup * csetup_
The instance of the problem to benchmark (if planning with controls)
Definition: Benchmark.h:337
unsigned int activeRun
The number of the run currently being executed.
Definition: Benchmark.h:71
const Status & getStatus() const
Get the status of the benchmarking code. This function can be called in a separate thread to check ho...
Definition: Benchmark.h:308
std::vector< base::PlannerPtr > planners_
The set of planners to be tested.
Definition: Benchmark.h:340
std::vector< std::string > progressPropertyNames
Definition: Benchmark.h:100
boost::uint32_t seed
The random seed that was used at the start of the benchmark program.
Definition: Benchmark.h:143
unsigned int runCount
The number of runs to execute for each planner.
Definition: Benchmark.h:131
double totalDuration
The amount of time spent to collect the information in this structure (seconds)
Definition: Benchmark.h:137
std::map< std::string, std::string > parameters
Additional, experiment specific parameters. This is optional.
Definition: Benchmark.h:152
std::function< void(const base::PlannerPtr &)> PreSetupEvent
Signature of function that can be called before a planner execution is started.
Definition: Benchmark.h:84
virtual bool saveResultsToStream(std::ostream &out=std::cout) const
Save the results of the benchmark to a stream.
Definition: Benchmark.cpp:263
Create the set of classes typically needed to solve a control problem.
Definition: SimpleSetup.h:63
std::vector< PlannerExperiment > planners
The collected experimental data; each element of the array (an experiment) corresponds to a planner...
Definition: Benchmark.h:122
double maxMem
the maximum amount of memory a planner is allowed to use (MB); 4096.0 by default
Definition: Benchmark.h:177
double timeBetweenUpdates
When collecting time-varying data from a planner during its execution, the planner&#39;s progress will be...
Definition: Benchmark.h:184
void addPlanner(const base::PlannerPtr &planner)
Add a planner to use.
Definition: Benchmark.h:253
std::string name
The name of the planner.
Definition: Benchmark.h:93
std::map< std::string, std::string > RunProperties
The data collected from a run of a planner is stored as key-value pairs.
Definition: Benchmark.h:79
const base::SpaceInformationPtr & getSpaceInformation() const
Get the current instance of the space information.
Definition: SimpleSetup.h:75
void setExperimentName(const std::string &name)
Set the name of the experiment.
Definition: Benchmark.h:241
std::size_t numExperimentParameters() const
Return the number of optional benchmark parameters.
Definition: Benchmark.h:235
Benchmark a set of planners on a problem instance.
Definition: Benchmark.h:48
const std::map< std::string, std::string > & getExperimentParameters() const
Get all optional benchmark parameters. The map key is &#39;name type&#39;.
Definition: Benchmark.h:229
Benchmark(control::SimpleSetup &setup, const std::string &name=std::string())
Constructor needs the SimpleSetup instance needed for planning. Optionally, the experiment name (name...
Definition: Benchmark.h:212
bool displayProgress
flag indicating whether progress is to be displayed or not; true by default
Definition: Benchmark.h:187
RunProperties common
Some common properties for all the runs.
Definition: Benchmark.h:107
unsigned int runCount
the number of times to run each planner; 100 by default
Definition: Benchmark.h:180
time::point startTime
The point in time when the experiment was started.
Definition: Benchmark.h:134
bool saveConsoleOutput
flag indicating whether console output is saved (in an automatically generated filename); true by def...
Definition: Benchmark.h:191
bool useThreads
flag indicating whether planner runs should be run in a separate thread. It is advisable to set this ...
Definition: Benchmark.h:196
void addPlannerAllocator(const base::PlannerAllocator &pa)
Add a planner allocator to use.
Definition: Benchmark.h:263
Main namespace. Contains everything in this library.
Definition: AppBase.h:21
geometric::SimpleSetup * gsetup_
The instance of the problem to benchmark (if geometric planning)
Definition: Benchmark.h:334
Create the set of classes typically needed to solve a geometric problem.
Definition: SimpleSetup.h:63
Benchmark(geometric::SimpleSetup &setup, const std::string &name=std::string())
Constructor needs the SimpleSetup instance needed for planning. Optionally, the experiment name (name...
Definition: Benchmark.h:204
A shared pointer wrapper for ompl::base::Planner.
Request(double maxTime=5.0, double maxMem=4096.0, unsigned int runCount=100, double timeBetweenUpdates=0.05, bool displayProgress=true, bool saveConsoleOutput=true, bool useThreads=true, bool simplify=true)
Constructor that provides default values for all members.
Definition: Benchmark.h:159
std::vector< RunProperties > runs
Data collected for each run.
Definition: Benchmark.h:96
std::vector< RunProgressData > runsProgressData
Definition: Benchmark.h:104
double maxTime
the maximum amount of time a planner is allowed to run (seconds); 5.0 by default
Definition: Benchmark.h:174
void addExperimentParameter(const std::string &name, const std::string &type, const std::string &value)
Add an optional parameter&#39;s information to the benchmark output. Useful for aggregating results over ...
Definition: Benchmark.h:223
This structure contains information about the activity of a benchmark instance. If the instance is ru...
Definition: Benchmark.h:55
double maxMem
The maximum allowed memory for planner computation during the experiment (MB)
Definition: Benchmark.h:128
PostSetupEvent postRun_
Event to be called after the run of a planner.
Definition: Benchmark.h:355
std::function< void(const base::PlannerPtr &, RunProperties &)> PostSetupEvent
Signature of function that can be called after a planner execution is completed.
Definition: Benchmark.h:87
const CompleteExperiment & getRecordedExperimentData() const
Return all the experiment data that would be written to the results file. The data should not be chan...
Definition: Benchmark.h:317
void setPreRunEvent(const PreSetupEvent &event)
Set the event to be called before the run of a planner.
Definition: Benchmark.h:281
void clearPlanners()
Clear the set of planners to be benchmarked.
Definition: Benchmark.h:269
PreSetupEvent plannerSwitch_
Event to be called when the evaluated planner is switched.
Definition: Benchmark.h:349
The exception type for ompl.
Definition: Exception.h:46
The data collected after running a planner multiple times.
Definition: Benchmark.h:90
PreSetupEvent preRun_
Event to be called before the run of a planner.
Definition: Benchmark.h:352
Representation of a benchmark request.
Definition: Benchmark.h:156
void setPlannerSwitchEvent(const PreSetupEvent &event)
Set the event to be called before any runs of a particular planner (when the planner is switched) ...
Definition: Benchmark.h:275
bool simplify
flag indicating whether simplification should be applied to path; true by default ...
Definition: Benchmark.h:199
const SpaceInformationPtr & getSpaceInformation() const
Get the current instance of the space information.
Definition: SimpleSetup.h:75
bool saveResultsToFile() const
Save the results of the benchmark to a file. The name of the file is the current date and time...
Definition: Benchmark.cpp:257
std::string host
Hostname that identifies the machine the benchmark ran on.
Definition: Benchmark.h:146
std::string setupInfo
The output of SimpleSetup::print() before the experiment was started.
Definition: Benchmark.h:140
std::function< PlannerPtr(const SpaceInformationPtr &)> PlannerAllocator
Definition of a function that can allocate a planner.
Definition: Planner.h:433
std::chrono::system_clock::time_point point
Representation of a point in time.
Definition: Time.h:64
std::string name
The name of the experiment.
Definition: Benchmark.h:119
double progressPercentage
Total progress (0 to 100)
Definition: Benchmark.h:74
Status status_
The current status of this benchmarking instance.
Definition: Benchmark.h:346
void setPostRunEvent(const PostSetupEvent &event)
Set the event to be called after the run of a planner.
Definition: Benchmark.h:287
CompleteExperiment exp_
The collected experimental data (for all planners)
Definition: Benchmark.h:343
This structure holds experimental data for a set of planners.
Definition: Benchmark.h:116
std::string cpuInfo
Information about the CPU of the machine the benchmark ran on.
Definition: Benchmark.h:149
std::string activePlanner
The name of the planner currently being tested.
Definition: Benchmark.h:68
virtual void benchmark(const Request &req)
Benchmark the added planners on the defined problem. Repeated calls clear previously gathered data...
Definition: Benchmark.cpp:392