37 #include "ompl/tools/benchmark/Benchmark.h"
38 #include "ompl/tools/benchmark/MachineSpecs.h"
39 #include "ompl/util/Time.h"
40 #include "ompl/config.h"
41 #include "ompl/util/String.h"
42 #include <boost/scoped_ptr.hpp>
43 #include <boost/progress.hpp>
46 #include <condition_variable>
57 static std::string getResultsFilename(
const Benchmark::CompleteExperiment &exp)
59 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".log";
64 static std::string getConsoleFilename(
const Benchmark::CompleteExperiment &exp)
66 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".console";
79 RunPlanner(
const Benchmark *benchmark,
bool useThreads)
80 : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0), useThreads_(useThreads)
126 double getTimeUsed()
const
136 base::PlannerStatus getStatus()
const
141 const Benchmark::RunProgressData &getRunProgressData()
const
143 return runProgressData_;
157 return terminationCondition(maxMem, endtime);
167 boost::scoped_ptr<std::thread> t;
168 if (planner->getPlannerProgressProperties().size() > 0)
169 t.reset(
new std::thread([
this, &planner, timeBetweenUpdates]
171 collectProgressProperties(planner->getPlannerProgressProperties(),
174 status_ = planner->solve(ptc, 0.1);
177 solvedCondition_.notify_all();
178 solvedFlag_.unlock();
182 catch (std::runtime_error &e)
184 std::stringstream es;
185 es <<
"There was an error executing planner " << benchmark_->getStatus().activePlanner
186 <<
", run = " << benchmark_->getStatus().activeRun << std::endl;
187 es <<
"*** " << e.what() << std::endl;
188 std::cerr << es.str();
201 std::unique_lock<std::mutex> ulock(solvedFlag_);
204 if (solvedCondition_.wait_for(ulock, timePerUpdate) == std::cv_status::no_timeout)
210 std::map<std::string, std::string> data;
211 data[
"time REAL"] = timeStamp;
212 for (
const auto &property : properties)
214 data[
property.first] =
property.second();
216 runProgressData_.push_back(data);
221 const Benchmark *benchmark_;
224 base::PlannerStatus status_;
226 Benchmark::RunProgressData runProgressData_;
230 std::mutex solvedFlag_;
231 std::condition_variable solvedCondition_;
241 std::ofstream fout(filename);
250 if (getResultsFilename(
exp_) != std::string(filename))
253 OMPL_ERROR(
"Unable to write results to '%s'", filename);
260 std::string filename = getResultsFilename(exp_);
261 return saveResultsToFile(filename.c_str());
266 if (exp_.planners.empty())
268 OMPL_WARN(
"There is no experimental data to save");
278 out <<
"OMPL version " << OMPL_VERSION << std::endl;
279 out <<
"Experiment " << (exp_.name.empty() ?
"NO_NAME" : exp_.name) << std::endl;
281 out << exp_.parameters.size() <<
" experiment properties" << std::endl;
282 for (
const auto ¶meter : exp_.parameters)
283 out << parameter.first <<
" = " << parameter.second << std::endl;
285 out <<
"Running on " << (exp_.host.empty() ?
"UNKNOWN" : exp_.host) << std::endl;
287 out <<
"<<<|" << std::endl << exp_.setupInfo <<
"|>>>" << std::endl;
288 out <<
"<<<|" << std::endl << exp_.cpuInfo <<
"|>>>" << std::endl;
290 out << exp_.seed <<
" is the random seed" << std::endl;
291 out << exp_.maxTime <<
" seconds per run" << std::endl;
292 out << exp_.maxMem <<
" MB per run" << std::endl;
293 out << exp_.runCount <<
" runs per planner" << std::endl;
294 out << exp_.totalDuration <<
" seconds spent to collect the data" << std::endl;
297 out <<
"1 enum type" << std::endl;
303 out << exp_.planners.size() <<
" planners" << std::endl;
305 for (
const auto &planner : exp_.planners)
307 out << planner.name << std::endl;
310 std::vector<std::string> properties;
311 for (
auto &property : planner.common)
312 properties.push_back(property.first);
313 std::sort(properties.begin(), properties.end());
316 out << properties.size() <<
" common properties" << std::endl;
317 for (
auto &property : properties)
319 auto it = planner.common.find(property);
320 out << it->first <<
" = " << it->second << std::endl;
324 std::map<std::string, bool> propSeen;
325 for (
auto &run : planner.runs)
326 for (
auto &property : run)
327 propSeen[
property.first] =
true;
331 for (
auto &it : propSeen)
332 properties.push_back(it.first);
333 std::sort(properties.begin(), properties.end());
336 out << properties.size() <<
" properties for each run" << std::endl;
337 for (
auto &property : properties)
338 out <<
property << std::endl;
341 out << planner.runs.size() <<
" runs" << std::endl;
342 for (
auto &run : planner.runs)
344 for (
auto &property : properties)
346 auto it = run.find(property);
355 if (planner.runsProgressData.size() > 0)
358 out << planner.progressPropertyNames.size() <<
" progress properties for each run" << std::endl;
360 for (
const auto &progPropName : planner.progressPropertyNames)
362 out << progPropName << std::endl;
365 out << planner.runsProgressData.size() <<
" runs" << std::endl;
366 for (
const auto &r : planner.runsProgressData)
369 for (
const auto &t : r)
372 for (
const auto &iter : t)
374 out << iter.second <<
",";
386 out <<
'.' << std::endl;
396 if (!gsetup_->getSpaceInformation()->isSetup())
397 gsetup_->getSpaceInformation()->setup();
401 if (!csetup_->getSpaceInformation()->isSetup())
402 csetup_->getSpaceInformation()->setup();
405 if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
411 if (planners_.empty())
413 OMPL_ERROR(
"There are no planners to benchmark");
417 status_.running =
true;
418 exp_.totalDuration = 0.0;
431 exp_.planners.clear();
432 exp_.planners.resize(planners_.size());
434 const base::ProblemDefinitionPtr &pdef =
435 gsetup_ ? gsetup_->getProblemDefinition() : csetup_->getProblemDefinition();
437 for (
unsigned int i = 0; i < planners_.size(); ++i)
440 planners_[i]->setProblemDefinition(pdef);
441 if (!planners_[i]->isSetup())
442 planners_[i]->setup();
443 exp_.planners[i].name = (gsetup_ ?
"geometric_" :
"control_") + planners_[i]->getName();
444 OMPL_INFORM(
"Configured %s", exp_.planners[i].name.c_str());
448 OMPL_INFORM(
"Saving planner setup information ...");
450 std::stringstream setupInfo;
452 gsetup_->print(setupInfo);
454 csetup_->print(setupInfo);
455 setupInfo << std::endl <<
"Properties of benchmarked planners:" << std::endl;
456 for (
auto &planner : planners_)
457 planner->printProperties(setupInfo);
459 exp_.setupInfo = setupInfo.str();
465 boost::scoped_ptr<msg::OutputHandlerFile> ohf;
475 boost::scoped_ptr<boost::progress_display> progress;
478 std::cout <<
"Running experiment " << exp_.name <<
"." << std::endl;
479 std::cout <<
"Each planner will be executed " << req.
runCount <<
" times for at most " << req.
maxTime
480 <<
" seconds. Memory is limited at " << req.
maxMem <<
"MB." << std::endl;
481 progress.reset(
new boost::progress_display(100, std::cout));
487 for (
unsigned int i = 0; i < planners_.size(); ++i)
489 status_.activePlanner = exp_.planners[i].name;
495 OMPL_INFORM(
"Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
496 plannerSwitch_(planners_[i]);
497 OMPL_INFORM(
"Completed execution of planner-switch event");
500 catch (std::runtime_error &e)
502 std::stringstream es;
503 es <<
"There was an error executing the planner-switch event for planner " << status_.activePlanner
505 es <<
"*** " << e.what() << std::endl;
506 std::cerr << es.str();
513 planners_[i]->params().getParams(exp_.planners[i].common);
514 planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
517 exp_.planners[i].progressPropertyNames.emplace_back(
"time REAL");
518 base::Planner::PlannerProgressProperties::const_iterator iter;
519 for (iter = planners_[i]->getPlannerProgressProperties().begin();
520 iter != planners_[i]->getPlannerProgressProperties().end(); ++iter)
522 exp_.planners[i].progressPropertyNames.push_back(iter->first);
524 std::sort(exp_.planners[i].progressPropertyNames.begin(), exp_.planners[i].progressPropertyNames.end());
527 for (
unsigned int j = 0; j < req.
runCount; ++j)
529 status_.activeRun = j;
530 status_.progressPercentage =
531 (double)(100 * (req.
runCount * i + j)) / (
double)(planners_.size() * req.
runCount);
534 while (status_.progressPercentage > progress->count())
537 OMPL_INFORM(
"Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
542 planners_[i]->clear();
545 gsetup_->getProblemDefinition()->clearSolutionPaths();
546 gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
550 csetup_->getProblemDefinition()->clearSolutionPaths();
551 csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
554 catch (std::runtime_error &e)
556 std::stringstream es;
557 es <<
"There was an error while preparing for run " << status_.activeRun <<
" of planner "
558 << status_.activePlanner << std::endl;
559 es <<
"*** " << e.what() << std::endl;
560 std::cerr << es.str();
569 OMPL_INFORM(
"Executing pre-run event for run %d of planner %s ...", status_.activeRun,
570 status_.activePlanner.c_str());
571 preRun_(planners_[i]);
572 OMPL_INFORM(
"Completed execution of pre-run event");
575 catch (std::runtime_error &e)
577 std::stringstream es;
578 es <<
"There was an error executing the pre-run event for run " << status_.activeRun <<
" of planner "
579 << status_.activePlanner << std::endl;
580 es <<
"*** " << e.what() << std::endl;
581 std::cerr << es.str();
587 bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
595 run[
"memory REAL"] =
ompl::toString((
double)rp.getMemUsed() / (1024.0 * 1024.0));
599 run[
"solved BOOLEAN"] = std::to_string(gsetup_->haveExactSolutionPath());
600 run[
"valid segment fraction REAL"] =
601 ompl::toString(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
605 run[
"solved BOOLEAN"] = std::to_string(csetup_->haveExactSolutionPath());
606 run[
"valid segment fraction REAL"] =
607 ompl::toString(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
614 run[
"approximate solution BOOLEAN"] =
615 std::to_string(gsetup_->getProblemDefinition()->hasApproximateSolution());
616 run[
"solution difference REAL"] =
617 ompl::toString(gsetup_->getProblemDefinition()->getSolutionDifference());
618 run[
"solution length REAL"] =
ompl::toString(gsetup_->getSolutionPath().length());
619 run[
"solution smoothness REAL"] =
ompl::toString(gsetup_->getSolutionPath().smoothness());
620 run[
"solution clearance REAL"] =
ompl::toString(gsetup_->getSolutionPath().clearance());
621 run[
"solution segments INTEGER"] =
622 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
623 run[
"correct solution BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
625 unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
626 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
627 run[
"correct solution strict BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
628 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
634 gsetup_->simplifySolution();
637 run[
"simplified solution length REAL"] =
639 run[
"simplified solution smoothness REAL"] =
641 run[
"simplified solution clearance REAL"] =
643 run[
"simplified solution segments INTEGER"] =
644 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
645 run[
"simplified correct solution BOOLEAN"] =
646 std::to_string(gsetup_->getSolutionPath().check());
647 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
648 run[
"simplified correct solution strict BOOLEAN"] =
649 std::to_string(gsetup_->getSolutionPath().check());
650 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
655 run[
"approximate solution BOOLEAN"] =
656 std::to_string(csetup_->getProblemDefinition()->hasApproximateSolution());
657 run[
"solution difference REAL"] =
658 ompl::toString(csetup_->getProblemDefinition()->getSolutionDifference());
659 run[
"solution length REAL"] =
ompl::toString(csetup_->getSolutionPath().length());
660 run[
"solution clearance REAL"] =
661 ompl::toString(csetup_->getSolutionPath().asGeometric().clearance());
662 run[
"solution segments INTEGER"] = std::to_string(csetup_->getSolutionPath().getControlCount());
663 run[
"correct solution BOOLEAN"] = std::to_string(csetup_->getSolutionPath().check());
667 base::PlannerData pd(gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
668 planners_[i]->getPlannerData(pd);
669 run[
"graph states INTEGER"] = std::to_string(pd.
numVertices());
670 run[
"graph motions INTEGER"] = std::to_string(pd.
numEdges());
673 run[prop.first] = prop.second;
680 OMPL_INFORM(
"Executing post-run event for run %d of planner %s ...", status_.activeRun,
681 status_.activePlanner.c_str());
682 postRun_(planners_[i], run);
683 OMPL_INFORM(
"Completed execution of post-run event");
686 catch (std::runtime_error &e)
688 std::stringstream es;
689 es <<
"There was an error in the execution of the post-run event for run " << status_.activeRun
690 <<
" of planner " << status_.activePlanner << std::endl;
691 es <<
"*** " << e.what() << std::endl;
692 std::cerr << es.str();
696 exp_.planners[i].runs.push_back(run);
700 if (planners_[i]->getPlannerProgressProperties().size() > 0)
702 exp_.planners[i].runsProgressData.push_back(rp.getRunProgressData());
705 catch (std::runtime_error &e)
707 std::stringstream es;
708 es <<
"There was an error in the extraction of planner results: planner = " << status_.activePlanner
709 <<
", run = " << status_.activePlanner << std::endl;
710 es <<
"*** " << e.what() << std::endl;
711 std::cerr << es.str();
717 status_.running =
false;
718 status_.progressPercentage = 100.0;
721 while (status_.progressPercentage > progress->count())
723 std::cout << std::endl;