Benchmark.h

/*********************************************************************
 * Software License Agreement (BSD License)
 *
 *  Copyright (c) 2010, Rice University
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   * Neither the name of the Rice University nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 *********************************************************************/
 
/* Author: Ioan Sucan */
 
#ifndef OMPL_TOOLS_BENCHMARK_BENCHMARK_
#define OMPL_TOOLS_BENCHMARK_BENCHMARK_
 
#include "ompl/geometric/SimpleSetup.h"
#include "ompl/control/SimpleSetup.h"
 
namespace ompl
{
    namespace tools
    {
        class Benchmark
        {
        public:
            struct Status
            {
                bool running{false};
 
                unsigned int activeRun{0};
 
                double progressPercentage{0.};
 
                std::string activePlanner;
            };
 
            struct RunProperties : std::map<std::string, std::string>
            {
            };
 
            using RunProgressData = std::vector<std::map<std::string, std::string>>;
 
            using PreSetupEvent = std::function<void(const base::PlannerPtr &)>;
 
            using PostSetupEvent = std::function<void(const base::PlannerPtr &, RunProperties &)>;
 
            struct PlannerExperiment
            {
                std::string name;
 
                std::vector<RunProperties> runs;
 
                std::vector<std::string> progressPropertyNames;
 
                std::vector<RunProgressData> runsProgressData;
 
                RunProperties common;
 
                bool operator==(const PlannerExperiment &p) const
                {
                    return name == p.name && runs == p.runs && common == p.common;
                }
            };
 
            struct CompleteExperiment
            {
                std::string name;
 
                std::vector<PlannerExperiment> planners;
 
                double maxTime;
 
                double maxMem;
 
                unsigned int runCount;
 
                time::point startTime;
 
                double totalDuration;
 
                std::string setupInfo;
 
                std::uint_fast32_t seed;
 
                std::string host;
 
                std::string cpuInfo;
 
                std::map<std::string, std::string> parameters;
            };
 
            struct Request
            {
                Request(double maxTime = 5.0, double maxMem = 4096.0, unsigned int runCount = 100,
                        double timeBetweenUpdates = 0.05, bool displayProgress = true, bool saveConsoleOutput = true,
                        bool simplify = true)
                  : maxTime(maxTime)
                  , maxMem(maxMem)
                  , runCount(runCount)
                  , timeBetweenUpdates(timeBetweenUpdates)
                  , displayProgress(displayProgress)
                  , saveConsoleOutput(saveConsoleOutput)
                  , simplify(simplify)
                {
                }
 
                double maxTime;
 
                double maxMem;
 
                unsigned int runCount;
 
                double timeBetweenUpdates;
 
                bool displayProgress;
 
                bool saveConsoleOutput;
 
                bool simplify;
            };
 
            Benchmark(geometric::SimpleSetup &setup, const std::string &name = std::string());
 
            Benchmark(control::SimpleSetup &setup, const std::string &name = std::string());
 
            virtual ~Benchmark() = default;
 
            void addExperimentParameter(const std::string &name, const std::string &type, const std::string &value);
 
            const std::map<std::string, std::string> &getExperimentParameters() const;
 
            std::size_t numExperimentParameters() const;
 
            void setExperimentName(const std::string &name);
 
            const std::string &getExperimentName() const;
 
            void addPlanner(const base::PlannerPtr &planner);
 
            void addPlannerAllocator(const base::PlannerAllocator &pa);
 
            void clearPlanners();
 
            void setPlannerSwitchEvent(const PreSetupEvent &event);
 
            void setPreRunEvent(const PreSetupEvent &event);
 
            void setPostRunEvent(const PostSetupEvent &event);
 
            virtual void benchmark(const Request &req);
 
            const Status &getStatus() const;
 
            const CompleteExperiment &getRecordedExperimentData() const;
 
            virtual bool saveResultsToStream(std::ostream &out = std::cout) const;
 
            bool saveResultsToFile(const char *filename) const;
 
            bool saveResultsToFile() const;
 
        protected:
            geometric::SimpleSetup *gsetup_;
 
            control::SimpleSetup *csetup_;
 
            std::vector<base::PlannerPtr> planners_;
 
            CompleteExperiment exp_;
 
            Status status_;
 
            PreSetupEvent plannerSwitch_;
 
            PreSetupEvent preRun_;
 
            PostSetupEvent postRun_;
        };
    }  // namespace tools
}  // namespace ompl
#endif