Benchmark.h

/*********************************************************************
 * Software License Agreement (BSD License)
 *
 *  Copyright (c) 2010, Rice University
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   * Neither the name of the Rice University nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 *********************************************************************/
 
/* Author: Ioan Sucan */
 
#ifndef OMPL_TOOLS_BENCHMARK_BENCHMARK_
#define OMPL_TOOLS_BENCHMARK_BENCHMARK_
 
#include "ompl/geometric/SimpleSetup.h"
#include "ompl/control/SimpleSetup.h"
 
namespace ompl
{
    namespace tools
    {
        class Benchmark
        {
        public:
            struct Status
            {
                bool running{false};

                unsigned int activeRun{0};

                double progressPercentage{0.};

                std::string activePlanner;
            };

            struct RunProperties : std::map<std::string, std::string>
            {
            };
 
            using RunProgressData = std::vector<std::map<std::string, std::string>>;

            using PreSetupEvent = std::function<void(const base::PlannerPtr &)>;

            using PostSetupEvent = std::function<void(const base::PlannerPtr &, RunProperties &)>;

            struct PlannerExperiment
            {
                std::string name;

                std::vector<RunProperties> runs;

                std::vector<std::string> progressPropertyNames;

                std::vector<RunProgressData> runsProgressData;

                RunProperties common;
 
                bool operator==(const PlannerExperiment &p) const
                {
                    return name == p.name && runs == p.runs && common == p.common;
                }
            };

            struct CompleteExperiment
            {
                std::string name;

                std::vector<PlannerExperiment> planners;

                double maxTime;

                double maxMem;

                unsigned int runCount;

                time::point startTime;

                double totalDuration;

                std::string setupInfo;

                std::uint_fast32_t seed;

                std::string host;

                std::string cpuInfo;

                std::map<std::string, std::string> parameters;
            };

            struct Request
            {
                Request(double maxTime = 5.0, double maxMem = 4096.0, unsigned int runCount = 100,
                        double timeBetweenUpdates = 0.05, bool displayProgress = true, bool saveConsoleOutput = true,
                        bool simplify = true)
                  : maxTime(maxTime)
                  , maxMem(maxMem)
                  , runCount(runCount)
                  , timeBetweenUpdates(timeBetweenUpdates)
                  , displayProgress(displayProgress)
                  , saveConsoleOutput(saveConsoleOutput)
                  , simplify(simplify)
                {
                }

                double maxTime;

                double maxMem;

                unsigned int runCount;

                double timeBetweenUpdates;

                bool displayProgress;

                bool saveConsoleOutput;

                bool simplify;
            };

            Benchmark(geometric::SimpleSetup &setup, const std::string &name = std::string());

            Benchmark(control::SimpleSetup &setup, const std::string &name = std::string());
 
            virtual ~Benchmark() = default;

            void addExperimentParameter(const std::string &name, const std::string &type, const std::string &value);

            const std::map<std::string, std::string> &getExperimentParameters() const;

            std::size_t numExperimentParameters() const;

            void setExperimentName(const std::string &name);

            const std::string &getExperimentName() const;

            void addPlanner(const base::PlannerPtr &planner);

            void addPlannerAllocator(const base::PlannerAllocator &pa);

            void clearPlanners();

            void setPlannerSwitchEvent(const PreSetupEvent &event);

            void setPreRunEvent(const PreSetupEvent &event);

            void setPostRunEvent(const PostSetupEvent &event);

            virtual void benchmark(const Request &req);

            const Status &getStatus() const;

            const CompleteExperiment &getRecordedExperimentData() const;

            virtual bool saveResultsToStream(std::ostream &out = std::cout) const;

            bool saveResultsToFile(const char *filename) const;

            bool saveResultsToFile() const;
 
        protected:
            geometric::SimpleSetup *gsetup_;

            control::SimpleSetup *csetup_;

            std::vector<base::PlannerPtr> planners_;

            CompleteExperiment exp_;

            Status status_;

            PreSetupEvent plannerSwitch_;

            PreSetupEvent preRun_;

            PostSetupEvent postRun_;
        };
    }  // namespace tools
}  // namespace ompl
#endif