1 /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
3 #include "base/application.hpp"
4 #include "base/console.hpp"
5 #include "base/convert.hpp"
6 #include "base/json.hpp"
7 #include "base/netstring.hpp"
8 #include "base/objectlock.hpp"
9 #include "base/stdiostream.hpp"
10 #include "cli/daemonutility.hpp"
11 #include "cli/featureutility.hpp"
12 #include "cli/objectlistutility.hpp"
13 #include "cli/troubleshootcommand.hpp"
14 #include "cli/variableutility.hpp"
15 #include "config/configitembuilder.hpp"
17 #include <boost/algorithm/string/join.hpp>
18 #include <boost/circular_buffer.hpp>
19 #include <boost/filesystem.hpp>
24 using namespace icinga;
25 namespace po = boost::program_options;
27 REGISTER_CLICOMMAND("troubleshoot", TroubleshootCommand);
29 String TroubleshootCommand::GetDescription() const
31 return "Collect logs and other relevant information for troubleshooting purposes.";
34 String TroubleshootCommand::GetShortDescription() const
36 return "collect information for troubleshooting";
39 class TroubleshootCommand::InfoLog
42 InfoLog(const String& path, const bool cons)
45 m_ConsoleType = Console_Dumb;
47 m_Stream = new std::ostream(std::cout.rdbuf());
49 m_ConsoleType = Console_VT100;
51 m_ConsoleType = Console_Windows;
55 auto *ofs = new std::ofstream();
56 ofs->open(path.CStr(), std::ios::out | std::ios::trunc);
66 void WriteLine(const LogSeverity sev, const int color, const String& str)
69 Log(sev, "troubleshoot", str);
71 if (sev == LogWarning) {
73 << '\n' << ConsoleColorTag(Console_ForegroundYellow, m_ConsoleType) << std::string(24, '#') << '\n'
74 << ConsoleColorTag(Console_Normal, m_ConsoleType) << str
75 << ConsoleColorTag(Console_ForegroundYellow, m_ConsoleType) << std::string(24, '#') << "\n\n"
76 << ConsoleColorTag(Console_Normal, m_ConsoleType);
77 } else if (sev == LogCritical) {
79 << '\n' << ConsoleColorTag(Console_ForegroundRed, m_ConsoleType) << std::string(24, '#') << '\n'
80 << ConsoleColorTag(Console_Normal, m_ConsoleType) << str
81 << ConsoleColorTag(Console_ForegroundRed, m_ConsoleType) << std::string(24, '#') << "\n\n"
82 << ConsoleColorTag(Console_Normal, m_ConsoleType);
85 << ConsoleColorTag(color, m_ConsoleType) << str
86 << ConsoleColorTag(Console_Normal, m_ConsoleType);
89 bool GetStreamHealth() const
91 return m_Stream->good();
96 ConsoleType m_ConsoleType;
97 std::ostream *m_Stream;
100 class TroubleshootCommand::InfoLogLine
103 InfoLogLine(InfoLog& log, int col = Console_Normal, LogSeverity sev = LogInformation)
104 : m_Log(log), m_Color(col), m_Sev(sev) {}
108 m_Log.WriteLine(m_Sev, m_Color, m_String.str());
111 template <typename T>
112 InfoLogLine& operator<<(const T& info)
119 std::ostringstream m_String;
126 bool TroubleshootCommand::GeneralInfo(InfoLog& log, const boost::program_options::variables_map& vm)
128 InfoLogLine(log, Console_ForegroundBlue)
129 << std::string(14, '=') << " GENERAL INFORMATION " << std::string(14, '=') << "\n\n";
131 //Application::DisplayInfoMessage() but formatted
133 << "\tApplication version: " << Application::GetAppVersion() << '\n'
135 << "\tConfig directory: " << Configuration::ConfigDir << "\n"
136 << "\tData directory: " << Configuration::DataDir << "\n"
137 << "\tLog directory: " << Configuration::LogDir << "\n"
138 << "\tCache directory: " << Configuration::CacheDir << "\n"
139 << "\tRun directory: " << Configuration::InitRunDir << "\n"
141 << "Old paths (deprecated):\n"
142 << "\tInstallation root: " << Configuration::PrefixDir << '\n'
143 << "\tSysconf directory: " << Configuration::SysconfDir << '\n'
144 << "\tRun directory: " << Configuration::RunDir << '\n'
145 << "\tLocal state directory: " << Configuration::LocalStateDir << '\n'
147 << "Internal paths:\n"
148 << "\tPackage data directory: " << Configuration::PkgDataDir << '\n'
149 << "\tState path: " << Configuration::StatePath << '\n'
150 << "\tObjects path: " << Configuration::ObjectsPath << '\n'
151 << "\tVars path: " << Configuration::VarsPath << '\n'
152 << "\tPID path: " << Configuration::PidPath << '\n';
160 bool TroubleshootCommand::FeatureInfo(InfoLog& log, const boost::program_options::variables_map& vm)
162 TroubleshootCommand::CheckFeatures(log);
163 //TODO Check whether active features are operational.
167 bool TroubleshootCommand::ObjectInfo(InfoLog& log, const boost::program_options::variables_map& vm, Dictionary::Ptr& logs, const String& path)
169 InfoLogLine(log, Console_ForegroundBlue)
170 << std::string(14, '=') << " OBJECT INFORMATION " << std::string(14, '=') << "\n\n";
172 String objectfile = Configuration::ObjectsPath;
173 std::set<String> configs;
175 if (!Utility::PathExists(objectfile)) {
176 InfoLogLine(log, 0, LogCritical)
177 << "Cannot open object file '" << objectfile << "'.\n"
178 << "FAILED: This probably means you have a fault configuration.\n";
181 InfoLog *OFile = nullptr;
182 bool OConsole = false;
183 if (vm.count("include-objects")) {
184 if (vm.count("console"))
187 OFile = new InfoLog(path+"-objects", false);
188 if (!OFile->GetStreamHealth()) {
189 InfoLogLine(log, 0, LogWarning)
190 << "Failed to open Object-write-stream, not printing objects\n\n";
195 << "Printing all objects to " << path+"-objects\n";
198 CheckObjectFile(objectfile, log, OFile, OConsole, logs, configs);
202 if (vm.count("include-vars")) {
203 if (vm.count("console")) {
204 InfoLogLine(log, Console_ForegroundBlue)
205 << "\n[begin: varsfile]\n";
206 if (!PrintVarsFile(path, true))
207 InfoLogLine(log, 0, LogWarning)
208 << "Failed to print vars file\n";
209 InfoLogLine(log, Console_ForegroundBlue)
210 << "[end: varsfile]\n";
212 if (PrintVarsFile(path, false))
214 << "Successfully printed all variables to " << path+"-vars\n";
216 InfoLogLine(log, 0, LogWarning)
217 << "Failed to print vars to " << path+"-vars\n";
227 bool TroubleshootCommand::ReportInfo(InfoLog& log, const boost::program_options::variables_map& vm, Dictionary::Ptr& logs)
229 InfoLogLine(log, Console_ForegroundBlue)
230 << std::string(14, '=') << " LOGS AND CRASH REPORTS " << std::string(14, '=') << "\n\n";
231 PrintLoggers(log, logs);
232 PrintCrashReports(log);
240 bool TroubleshootCommand::ConfigInfo(InfoLog& log, const boost::program_options::variables_map& vm)
242 InfoLogLine(log, Console_ForegroundBlue)
243 << std::string(14, '=') << " CONFIGURATION FILES " << std::string(14, '=') << "\n\n";
246 << "A collection of important configuration files follows, please make sure to remove any sensitive data such as credentials, internal company names, etc\n";
248 if (!PrintFile(log, Configuration::ConfigDir + "/icinga2.conf")) {
249 InfoLogLine(log, 0, LogWarning)
250 << "icinga2.conf not found, therefore skipping validation.\n"
251 << "If you are using an icinga2.conf somewhere but the default path please validate it via 'icinga2 daemon -C -c \"path\to/icinga2.conf\"'\n"
252 << "and provide it with your support request.\n";
255 if (!PrintFile(log, Configuration::ConfigDir + "/zones.conf")) {
256 InfoLogLine(log, 0, LogWarning)
257 << "zones.conf not found.\n"
258 << "If you are using a zones.conf somewhere but the default path please provide it with your support request\n";
267 /*Print the last *numLines* of *file* to *os* */
268 int TroubleshootCommand::Tail(const String& file, int numLines, InfoLog& log)
270 boost::circular_buffer<std::string> ringBuf(numLines);
272 text.open(file.CStr(), std::ifstream::in);
279 while (std::getline(text, line)) {
280 ringBuf.push_back(line);
284 if (lines < numLines)
287 InfoLogLine(log, Console_ForegroundCyan)
288 << "[begin: '" << file << "' line: " << lines-numLines << "]\n";
290 for (int k = 0; k < numLines; k++) {
291 InfoLogLine(log, Console_ForegroundCyan)
294 << ringBuf[k] << '\n';
299 InfoLogLine(log, Console_ForegroundCyan)
300 << "[end: '" << file << "' line: " << lines << "]\n\n";
305 bool TroubleshootCommand::CheckFeatures(InfoLog& log)
307 Dictionary::Ptr features = new Dictionary;
308 std::vector<String> disabled_features;
309 std::vector<String> enabled_features;
311 if (!FeatureUtility::GetFeatures(disabled_features, true) ||
312 !FeatureUtility::GetFeatures(enabled_features, false)) {
313 InfoLogLine(log, 0, LogCritical)
314 << "Failed to collect enabled and/or disabled features. Check\n"
315 << FeatureUtility::GetFeaturesAvailablePath() << '\n'
316 << FeatureUtility::GetFeaturesEnabledPath() << '\n';
320 for (const String& feature : disabled_features)
321 features->Set(feature, false);
322 for (const String& feature : enabled_features)
323 features->Set(feature, true);
326 << "Enabled features:\n";
327 InfoLogLine(log, Console_ForegroundGreen)
328 << '\t' << boost::algorithm::join(enabled_features, " ") << '\n';
330 << "Disabled features:\n";
331 InfoLogLine(log, Console_ForegroundRed)
332 << '\t' << boost::algorithm::join(disabled_features, " ") << '\n';
334 if (!features->Get("checker").ToBool())
335 InfoLogLine(log, 0, LogWarning)
336 << "checker is disabled, no checks can be run from this instance\n";
337 if (!features->Get("mainlog").ToBool())
338 InfoLogLine(log, 0, LogWarning)
339 << "mainlog is disabled, please activate it and rerun icinga2\n";
340 if (!features->Get("debuglog").ToBool())
341 InfoLogLine(log, 0, LogWarning)
342 << "debuglog is disabled, please activate it and rerun icinga2\n";
347 void TroubleshootCommand::GetLatestReport(const String& filename, time_t& bestTimestamp, String& bestFilename)
351 if (_stat(filename.CStr(), &buf))
355 if (stat(filename.CStr(), &buf))
358 if (buf.st_mtime > bestTimestamp) {
359 bestTimestamp = buf.st_mtime;
360 bestFilename = filename;
364 bool TroubleshootCommand::PrintCrashReports(InfoLog& log)
366 String spath = Configuration::LogDir + "/crash/report.*";
367 time_t bestTimestamp = 0;
371 Utility::Glob(spath, std::bind(&GetLatestReport, _1, std::ref(bestTimestamp),
372 std::ref(bestFilename)), GlobFile);
375 catch (win32_error &ex) {
376 if (int const * err = boost::get_error_info<errinfo_win32_error>(ex)) {
377 if (*err != 3) {//Error code for path does not exist
378 InfoLogLine(log, 0, LogWarning)
379 << Configuration::LogDir + "/crash/ does not exist\n";
384 InfoLogLine(log, 0, LogWarning)
385 << "Error printing crash reports\n";
391 InfoLogLine(log, 0, LogWarning) << "Error printing crash reports.\n"
392 << "Does " << Configuration::LogDir + "/crash/ exist?\n";
399 InfoLogLine(log, Console_ForegroundYellow)
400 << "No crash logs found in " << Configuration::LogDir << "/crash/\n\n";
403 << "Latest crash report is from " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S", Utility::GetTime()) << '\n'
404 << "File: " << bestFilename << "\n\n";
405 PrintFile(log, bestFilename);
413 bool TroubleshootCommand::PrintFile(InfoLog& log, const String& path)
416 text.open(path.CStr(), std::ifstream::in);
422 InfoLogLine(log, Console_ForegroundCyan)
423 << "[begin: '" << path << "']\n";
425 while (std::getline(text, line)) {
426 InfoLogLine(log, Console_ForegroundCyan)
432 InfoLogLine(log, Console_ForegroundCyan)
433 << "[end: '" << path << "']\n";
438 bool TroubleshootCommand::CheckConfig()
440 String configDir = Configuration::ConfigDir;
441 String objectsPath = Configuration::ObjectsPath;
442 return DaemonUtility::ValidateConfigFiles({ configDir + "/icinga2.conf" }, objectsPath);
445 //print is supposed allow the user to print the object file
446 void TroubleshootCommand::CheckObjectFile(const String& objectfile, InfoLog& log, InfoLog *OFile, const bool objectConsole,
447 Dictionary::Ptr& logs, std::set<String>& configs)
450 << "Checking object file from " << objectfile << '\n';
453 fp.open(objectfile.CStr(), std::ios_base::in);
456 InfoLogLine(log, 0, LogWarning)
457 << "Could not open object file.\n";
461 StdioStream::Ptr sfp = new StdioStream(&fp, false);
462 String::SizeType typeL = 0, countTotal = 0;
465 StreamReadContext src;
466 StreamReadStatus srs;
467 std::map<String, int> type_count;
470 std::stringstream sStream;
473 InfoLogLine(log, Console_ForegroundBlue)
474 << "\n[begin: objectfile]\n";
476 while ((srs = NetString::ReadStringFromStream(sfp, &message, src)) != StatusEof) {
477 if (srs != StatusNewItem)
481 ObjectListUtility::PrintObject(std::cout, first, message, type_count, "", "");
484 ObjectListUtility::PrintObject(sStream, first, message, type_count, "", "");
492 Dictionary::Ptr object = JsonDecode(message);
493 Dictionary::Ptr properties = object->Get("properties");
495 String name = object->Get("name");
496 String type = object->Get("type");
498 //Find longest typename for padding
499 typeL = type.GetLength() > typeL ? type.GetLength() : typeL;
502 Array::Ptr debug_info = object->Get("debug_info");
505 configs.insert(debug_info->Get(0));
507 if (Utility::Match(type, "FileLogger")) {
508 Dictionary::Ptr debug_hints = object->Get("debug_hints");
509 Dictionary::Ptr properties = object->Get("properties");
511 ObjectLock olock(properties);
512 for (const Dictionary::Pair& kv : properties) {
513 if (Utility::Match(kv.first, "path"))
514 logs->Set(name, kv.second);
520 InfoLogLine(log, Console_ForegroundBlue)
521 << "\n[end: objectfile]\n";
524 InfoLogLine(log, 0, LogCritical)
525 << "No objects found in objectfile.\n";
529 //Print objects with count
531 << "Found the " << countTotal << " objects:\n"
532 << " Type" << std::string(typeL-4, ' ') << " : Count\n";
534 for (const Dictionary::Pair& kv : type_count) {
536 << " " << kv.first << std::string(typeL - kv.first.GetLength(), ' ')
537 << " : " << kv.second << '\n';
543 TroubleshootCommand::PrintObjectOrigin(log, configs);
546 bool TroubleshootCommand::PrintVarsFile(const String& path, const bool console) {
548 auto *ofs = new std::ofstream();
549 ofs->open((path+"-vars").CStr(), std::ios::out | std::ios::trunc);
553 VariableUtility::PrintVariables(*ofs);
556 VariableUtility::PrintVariables(std::cout);
561 void TroubleshootCommand::PrintLoggers(InfoLog& log, Dictionary::Ptr& logs)
563 if (!logs->GetLength()) {
564 InfoLogLine(log, 0, LogWarning)
565 << "No loggers found, check whether you enabled any logging features\n";
568 << "Getting the last 20 lines of " << logs->GetLength() << " FileLogger objects.\n";
570 ObjectLock ulock(logs);
571 for (const Dictionary::Pair& kv : logs) {
573 << "Logger " << kv.first << " at path: " << kv.second << '\n';
575 if (!Tail(kv.second, 20, log)) {
576 InfoLogLine(log, 0, LogWarning)
577 << kv.second << " either does not exist or is empty\n";
583 void TroubleshootCommand::PrintObjectOrigin(InfoLog& log, const std::set<String>& configSet)
586 << "The objects origins are:\n";
588 for (const String& config : configSet) {
590 << " " << config << '\n';
594 void TroubleshootCommand::InitParameters(boost::program_options::options_description& visibleDesc,
595 boost::program_options::options_description& hiddenDesc) const
597 visibleDesc.add_options()
598 ("console,c", "print to console instead of file")
599 ("output,o", boost::program_options::value<std::string>(), "path to output file")
600 ("include-objects", "Print the whole objectfile (like `object list`)")
601 ("include-vars", "Print all Variables (like `variable list`)")
605 int TroubleshootCommand::Run(const boost::program_options::variables_map& vm, const std::vector<std::string>& ap) const
607 #ifdef _WIN32 //Dislikes ':' in filenames
608 String path = Configuration::LogDir + "/troubleshooting-"
609 + Utility::FormatDateTime("%Y-%m-%d_%H-%M-%S", Utility::GetTime()) + ".log";
611 String path = Configuration::LogDir + "/troubleshooting-"
612 + Utility::FormatDateTime("%Y-%m-%d_%H:%M:%S", Utility::GetTime()) + ".log";
616 Logger::SetConsoleLogSeverity(LogWarning);
618 if (vm.count("output"))
619 path = vm["output"].as<std::string>();
621 if (vm.count("console")) {
622 log = new InfoLog("", true);
624 log = new InfoLog(path, false);
625 if (!log->GetStreamHealth()) {
626 Log(LogCritical, "troubleshoot", "Failed to open file to write: " + path);
632 String appName = Utility::BaseName(Application::GetArgV()[0]);
633 double goTime = Utility::GetTime();
636 << appName << " -- Troubleshooting help:\n"
637 << "Should you run into problems with Icinga please add this file to your help request\n"
638 << "Started collection at " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S", goTime) << "\n";
640 InfoLogLine(*log, Console_ForegroundMagenta)
641 << std::string(52, '=') << "\n\n";
643 if (appName.GetLength() > 3 && appName.SubStr(0, 3) == "lt-")
644 appName = appName.SubStr(3, appName.GetLength() - 3);
646 Dictionary::Ptr logs = new Dictionary;
648 if (!GeneralInfo(*log, vm) ||
649 !FeatureInfo(*log, vm) ||
650 !ObjectInfo(*log, vm, logs, path) ||
651 !ReportInfo(*log, vm, logs) ||
652 !ConfigInfo(*log, vm)) {
653 InfoLogLine(*log, 0, LogCritical)
654 << "Could not recover from critical failure, exiting.\n";
660 double endTime = Utility::GetTime();
662 InfoLogLine(*log, Console_ForegroundMagenta)
663 << std::string(52, '=') << '\n';
664 InfoLogLine(*log, Console_ForegroundGreen)
665 << "Finished collection at " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S", endTime)
666 << "\nTook " << Convert::ToString(endTime - goTime) << " seconds\n";
668 if (!vm.count("console")) {
669 std::cout << "Started collection at " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S", goTime) << "\n"
670 << "Finished collection at " << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S", endTime)
671 << "\nTook " << Convert::ToString(endTime - goTime) << " seconds\n\n";
673 std::cout << "General log file: '" << path << "'\n";
675 if (vm.count("include-vars"))
676 std::cout << "Vars log file: '" << path << "-vars'\n";
677 if (vm.count("include-objects"))
678 std::cout << "Objects log file: '" << path << "-objects'\n";
680 std::cout << "\nPlease compress the files before uploading them,, for example:\n"
681 << " # tar czf troubleshoot.tar.gz " << path << "*\n";