Benchmark.h

/*********************************************************************
* Software License Agreement (BSD License)
*
*  Copyright (c) 2010, Rice University
*  All rights reserved.
*
*  Redistribution and use in source and binary forms, with or without
*  modification, are permitted provided that the following conditions
*  are met:
*
*   * Redistributions of source code must retain the above copyright
*     notice, this list of conditions and the following disclaimer.
*   * Redistributions in binary form must reproduce the above
*     copyright notice, this list of conditions and the following
*     disclaimer in the documentation and/or other materials provided
*     with the distribution.
*   * Neither the name of the Rice University nor the names of its
*     contributors may be used to endorse or promote products derived
*     from this software without specific prior written permission.
*
*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
*  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
*  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
*  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
*  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
*  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
*  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
*  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
*  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
*  POSSIBILITY OF SUCH DAMAGE.
*********************************************************************/
 
/* Author: Ioan Sucan */
 
#ifndef OMPL_TOOLS_BENCHMARK_BENCHMARK_
#define OMPL_TOOLS_BENCHMARK_BENCHMARK_
 
#include "ompl/geometric/SimpleSetup.h"
#include "ompl/control/SimpleSetup.h"
 
namespace ompl
{
    namespace tools
    {
        class Benchmark
        {
        public:
            struct Status
            {
                Status()
                {
                    running = false;
                    activeRun = 0;
                    progressPercentage = 0.0;
                }
 
                bool running;
 
                std::string activePlanner;
 
                unsigned int activeRun;
 
                double progressPercentage;
            };
 
            using RunProperties = std::map<std::string, std::string>;
 
            using RunProgressData = std::vector<std::map<std::string, std::string>>;
 
            using PreSetupEvent = std::function<void(const base::PlannerPtr &)>;
 
            using PostSetupEvent = std::function<void(const base::PlannerPtr &, RunProperties &)>;
 
            struct PlannerExperiment
            {
                std::string name;
 
                std::vector<RunProperties> runs;
 
                std::vector<std::string> progressPropertyNames;
 
                std::vector<RunProgressData> runsProgressData;
 
                RunProperties common;
 
                bool operator==(const PlannerExperiment &p) const
                {
                    return name == p.name && runs == p.runs && common == p.common;
                }
            };
 
            struct CompleteExperiment
            {
                std::string name;
 
                std::vector<PlannerExperiment> planners;
 
                double maxTime;
 
                double maxMem;
 
                unsigned int runCount;
 
                time::point startTime;
 
                double totalDuration;
 
                std::string setupInfo;
 
                std::uint_fast32_t seed;
 
                std::string host;
 
                std::string cpuInfo;
 
                std::map<std::string, std::string> parameters;
            };
 
            struct Request
            {
                Request(double maxTime = 5.0, double maxMem = 4096.0, unsigned int runCount = 100,
                        double timeBetweenUpdates = 0.05, bool displayProgress = true, bool saveConsoleOutput = true,
                        bool simplify = true)
                  : maxTime(maxTime)
                  , maxMem(maxMem)
                  , runCount(runCount)
                  , timeBetweenUpdates(timeBetweenUpdates)
                  , displayProgress(displayProgress)
                  , saveConsoleOutput(saveConsoleOutput)
                  , simplify(simplify)
                {
                }
 
                double maxTime;
 
                double maxMem;
 
                unsigned int runCount;
 
                double timeBetweenUpdates;
 
                bool displayProgress;
 
                bool saveConsoleOutput;
 
                bool simplify;
            };
 
            Benchmark(geometric::SimpleSetup &setup, const std::string &name = std::string())
              : gsetup_(&setup), csetup_(nullptr)
            {
                exp_.name = name;
            }
 
            Benchmark(control::SimpleSetup &setup, const std::string &name = std::string())
              : gsetup_(nullptr), csetup_(&setup)
            {
                exp_.name = name;
            }
 
            virtual ~Benchmark() = default;
 
            void addExperimentParameter(const std::string &name, const std::string &type, const std::string &value)
            {
                exp_.parameters[name + " " + type] = value;
            }
 
            const std::map<std::string, std::string> &getExperimentParameters() const
            {
                return exp_.parameters;
            }
 
            std::size_t numExperimentParameters() const
            {
                return exp_.parameters.size();
            }
 
            void setExperimentName(const std::string &name)
            {
                exp_.name = name;
            }
 
            const std::string &getExperimentName() const
            {
                return exp_.name;
            }
 
            void addPlanner(const base::PlannerPtr &planner)
            {
                if (planner &&
                    planner->getSpaceInformation().get() !=
                        (gsetup_ != nullptr ? gsetup_->getSpaceInformation().get() : csetup_->getSpaceInformation().get()))
                    throw Exception("Planner instance does not match space information");
                planners_.push_back(planner);
            }
 
            void addPlannerAllocator(const base::PlannerAllocator &pa)
            {
                planners_.push_back(pa(gsetup_ != nullptr ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation()));
            }
 
            void clearPlanners()
            {
                planners_.clear();
            }
 
            void setPlannerSwitchEvent(const PreSetupEvent &event)
            {
                plannerSwitch_ = event;
            }
 
            void setPreRunEvent(const PreSetupEvent &event)
            {
                preRun_ = event;
            }
 
            void setPostRunEvent(const PostSetupEvent &event)
            {
                postRun_ = event;
            }
 
            virtual void benchmark(const Request &req);
 
            const Status &getStatus() const
            {
                return status_;
            }
 
            const CompleteExperiment &getRecordedExperimentData() const
            {
                return exp_;
            }
 
            virtual bool saveResultsToStream(std::ostream &out = std::cout) const;
 
            bool saveResultsToFile(const char *filename) const;
 
            bool saveResultsToFile() const;
 
        protected:
            geometric::SimpleSetup *gsetup_;
 
            control::SimpleSetup *csetup_;
 
            std::vector<base::PlannerPtr> planners_;
 
            CompleteExperiment exp_;
 
            Status status_;
 
            PreSetupEvent plannerSwitch_;
 
            PreSetupEvent preRun_;
 
            PostSetupEvent postRun_;
        };
    }
}
#endif