echo "Running"
else
echo "Not running"
+ exit 3
fi
}
#!/bin/sh
-sudo service httpd status
+sudo service httpd status &> /dev/null
+if [ $? -gt 0 ]; then
+ echo '[FAIL] httpd is not running'
+ exit 1
+else
+ echo '[OK] httpd is running'
+ exit 0
+fi
"""
+def success(msg):
+ print '[OK] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
+
+
+def fail(msg):
+ print '[FAIL] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
+
+
def main():
run_query = lambda q: utils.run_mysql_query(q, b'/usr/bin/mysql')
"where c.display_name = 'PassiveService1'"
state_time = float(next(iter(run_query(query)), {}).get('last_check', '0'))
if state_time == 0:
- print '"PassiveService1" seems not to have been checked yet'
+ fail('"PassiveService1" seems not to have been checked yet')
return 1
if (state_time + CHECK_INTERVAL) - time.time() < 30:
"where c.display_name = 'PassiveService1'"
output = next(iter(run_query(query)), {}).get('output', '')
if output != 'Passing in CheckResult header files works!':
- print 'Checkresult header files seem not to be processed properly'
+ fail('Checkresult header files seem not to be processed properly')
return 1
- print 'Checkresult header files are processed properly'
+ success('Checkresult header files are processed properly')
return 0
import os
import sys
import time
+import socket
import utils
+LIVESTATUS_PATH = '/var/run/icinga2/cmd/livestatus'
LS_HOST_COLUMNS = [
'name',
'name',
def success(msg):
print '[OK] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
def fail(msg):
print '[FAIL] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
def info(msg):
print '[INFO] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
+
+
+def error(msg):
+ print '[ERROR] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
def main():
if __name__ == '__main__':
- with utils.LiveStatusSocket('/var/run/icinga2/cmd/livestatus') as LIVESTATUS:
- sys.exit(main())
+ try:
+ with utils.LiveStatusSocket(LIVESTATUS_PATH) as LIVESTATUS:
+ sys.exit(main())
+ except (OSError, IOError, socket.error), e:
+ error('Could not connect to Livestatus socket: {0} ({1})'
+ ''.format(LIVESTATUS_PATH, unicode(e)))
+
if [ -e "/var/run/icinga2/cmd/icinga2.cmd" ];
then
- echo "Icinga2 commandpipe found"
+ echo "[OK] Icinga2 commandpipe found"
exit 0
else
- echo "Icinga2 commandpipe not found"
+ echo "[FAIL] Icinga2 commandpipe not found"
exit 1
fi
import sys
import time
import random
+import socket
import subprocess
try:
ICINGA_UID = 'icinga'
USERNAME = 'Icinga 2 Admin'
MAX_CHECK_ATTEMPTS = 3
+LIVESTATUS_PATH = '/var/run/icinga2/cmd/livestatus'
def send_command(command, quiet=False):
def success(msg):
print '[OK] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
return False
def fail(msg):
print '[FAIL] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
+ return True
+
+
+def error(msg):
+ print '[ERROR] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
return True
' INNER JOIN icinga_hosts AS h'
' ON h.host_object_id = c.host_object_id'
' WHERE h.alias = "{0}"'
- ''.format(hostname))['e_time'])
+ ''.format(hostname)).get('e_time', -1))
+ if execution_time == -1:
+ return fail('Unable to fetch the maximum execution time of'
+ ' host "{0}" from the IDO'.format(hostname))
send_command('DISABLE_HOST_CHECK;{0}'.format(hostname))
scheduled_check = time.time() + 4
' INNER JOIN icinga_hosts as h'
' ON h.host_object_id = s.host_object_id'
' WHERE h.alias = "{0}" AND s.display_name = "{1}"'
- ''.format(hostname, servicename))['e_time'])
+ ''.format(hostname, servicename)).get('e_time', -1))
+ if execution_time == -1:
+ return fail('Unable to fetch the maximum execution time of service "{0}"'
+ ' on host "{1}" from the IDO'.format(hostname, servicename))
send_command('DISABLE_SVC_CHECK;{0};{1}'.format(hostname, servicename))
scheduled_check = time.time() + 4
if __name__ == '__main__':
- with utils.LiveStatusSocket('/var/run/icinga2/cmd/livestatus') as LIVESTATUS:
- sys.exit(main())
+ try:
+ with utils.LiveStatusSocket(LIVESTATUS_PATH) as LIVESTATUS:
+ sys.exit(main())
+ except (OSError, IOError, socket.error), e:
+ error('Could not connect to Livestatus socket: {0} ({1})'
+ ''.format(LIVESTATUS_PATH, unicode(e)))
from __future__ import unicode_literals
+import sys
from datetime import datetime, timedelta
CHECK_INTERVAL = 10 # minutes; The actual interval are 5 minutes but as other
}
+def success(msg):
+ print '[OK] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
+ return True
+
+
+def fail(msg):
+ print '[FAIL] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
+ return False
+
+
+def info(msg):
+ print '[INFO] {0}'.format(msg).encode('utf-8')
+ sys.stdout.flush()
+
+
def validate_tables(tables):
"""
Return whether all tables of the IDO database scheme exist in
the given table listing
"""
- missing = [n for n in TABLES if TABLE_PREFIX + n not in tables]
- if missing:
- print 'Some tables are missing in the IDO'
- print 'Missing tables: ' + ', '.join(missing)
- return False
+ info('Checking database scheme... (tables)')
+ failures = False
+ for table in (TABLE_PREFIX + n for n in TABLES):
+ if table in tables:
+ success('Found table "{0}" in database'.format(table))
+ else:
+ fail('Could not find table "{0}" in database'.format(table))
+ failures = True
- print 'All tables were found in the IDO'
- return True
+ return not failures
def verify_host_config(config_data):
Return whether the example hosts exist in the given "hosts" table
"""
- if len([1 for e in config_data if e['alias'] in EXAMPLE_CONFIG]) == 2:
- print 'All example hosts are stored in the IDO'
- return True
+ info('Checking example host configuration...')
+ failures = False
+ for hostname in EXAMPLE_CONFIG:
+ if not any(1 for e in config_data if e['alias'] == hostname):
+ fail('Could not find host "{0}"'.format(hostname))
+ failures = True
+ else:
+ success('Found host "{0}"'.format(hostname))
- print 'Some example hosts are missing in the IDO'
- return False
+ return not failures
def verify_service_config(config_data):
Return whether the example services exist in the given "services" table
"""
+ info('Checking example service configuration...')
+ failures = False
for hostname, servicename in ((h, s) for h, ss in EXAMPLE_CONFIG.iteritems()
for s in ss):
- # Not very efficient, but suitable for just two hosts...
if not any(1 for c in config_data
if c['alias'] == hostname and
c['display_name'] == servicename):
- print 'The config stored in the IDO is missing some services'
- return False
+ fail('Could not find service "{0}" on host "{1}"'
+ ''.format(servicename, hostname))
+ failures = True
+ else:
+ success('Found service "{0}" on host "{1}"'
+ ''.format(servicename, hostname))
- print 'The service config stored in the IDO is correct'
- return True
+ return not failures
def check_last_host_status_update(check_info):
Return whether the example hosts are checked as scheduled
"""
- for info in check_info:
- if info['alias'] == 'localhost':
- last_check = datetime.fromtimestamp(float(info['last_check']))
+ info('Checking last host status updates...')
+ failures = False
+ for host_info in check_info:
+ if host_info['alias'] == 'localhost':
+ last_check = datetime.fromtimestamp(float(host_info['last_check']))
if datetime.now() - last_check > timedelta(minutes=CHECK_INTERVAL,
seconds=10):
- print 'The last status update of host "localhost"' \
- ' was more than {0} minutes ago'.format(CHECK_INTERVAL)
- return False
- elif info['alias'] == 'nsca-ng':
- if float(info['last_check']) > 0:
- print 'The host "nsca-ng" was checked even though' \
- ' it should not be actively checked'
- return False
-
- print 'The updates of both example hosts are processed as configured'
- return True
+ fail('The last status update of host "{0}" was more than {1} '
+ 'minutes ago'.format(host_info['alias'], CHECK_INTERVAL))
+ failures = True
+ else:
+ success('Host "{0}" is being updated'.format(host_info['alias']))
+ elif host_info['alias'] == 'nsca-ng':
+ if float(host_info['last_check']) > 0:
+ fail('The host "{0}" was checked even though it has'
+ ' no check service'.format(host_info['alias']))
+ failures = True
+ else:
+ success('Host "{0}" is not being checked because there '
+ 'is no check service'.format(host_info['alias']))
+ else:
+ info('Skipping host "{0}"'.format(host_info['alias']))
+
+ return not failures
def check_last_service_status_update(check_info):
Return whether the example services are checked as scheduled
"""
- for info in check_info:
- if info['display_name'] in EXAMPLE_CONFIG.get(info['alias'], []):
- last_check = datetime.fromtimestamp(float(info['last_check']))
+ info('Checking last service status updates...')
+ failures = False
+ for svc_info in check_info:
+ if svc_info['display_name'] in EXAMPLE_CONFIG.get(svc_info['alias'], []):
+ last_check = datetime.fromtimestamp(float(svc_info['last_check']))
if datetime.now() - last_check > timedelta(minutes=CHECK_INTERVAL,
seconds=10):
- print 'The last status update of service "{0}" of' \
- ' host "{1}" was more than {2} minutes ago' \
- ''.format(info['display_name'], info['alias'],
- CHECK_INTERVAL)
- return False
+ fail('The last status update of service "{0}" on'
+ ' host "{1}" was more than {2} minutes ago'
+ ''.format(svc_info['display_name'], svc_info['alias'],
+ CHECK_INTERVAL))
+ failures = True
+ else:
+ success('Service "{0}" on host "{1}" is being updated'
+ ''.format(svc_info['display_name'], svc_info['alias']))
+ else:
+ info('Skipping service "{0}" on host "{1}"'
+ ''.format(svc_info['display_name'], svc_info['alias']))
- print 'The updates of all example services are processed as configured'
- return True
+ return not failures
def check_logentries(logentry_info):
and refers to its very last hard status change
"""
+ info('Checking status log for host "localhost"...')
if logentry_info and logentry_info[0]['alias'] == 'localhost':
entry_time = datetime.fromtimestamp(float(logentry_info[0]['entry_time']))
state_time = datetime.fromtimestamp(float(logentry_info[0]['state_time']))
if entry_time - state_time > timedelta(seconds=10):
- print 'The last hard state of host "localhost"' \
- ' seems not to have been logged'
- return False
+ return fail('The last hard state of host "localhost"'
+ ' seems not to have been logged')
else:
- print 'No logs found in the IDO for host "localhost"'
- return False
+ return fail('No logs found in the IDO for host "localhost"')
- print 'The last hard state of host "localhost" was properly logged'
- return True
+ return success('The last hard state of host "localhost" was properly logged')
#!/bin/sh
-sudo service icinga2 status
+sudo service icinga2 status &> /dev/null
+if [ $? -gt 0 ]; then
+ echo '[FAIL] icinga2 is not running'
+ exit 1
+else
+ echo '[OK] icinga2 is running'
+ exit 0
+fi
def main():
+ failures = False
+
run_query = lambda q: utils.run_mysql_query(q, b'/usr/bin/mysql')
if not ido_tests.validate_tables([d['Tables_in_icinga']
for d in run_query('show tables')]):
- return 1
+ return 1 # Bail out as we cannot proceed without any data
host_info = run_query('select * from icinga_hosts')
if not ido_tests.verify_host_config(host_info):
- return 1
+ return 1 # Bail out as we cannot proceed without any data
service_info = run_query(
'select c2.alias, c1.* from icinga_services as c1 '
' on c1.host_object_id = c2.host_object_id'
)
if not ido_tests.verify_service_config(service_info):
- return 1
+ return 1 # Bail out as we cannot proceed without any data
hostchecks_data = run_query(
'select c.alias, unix_timestamp(s.last_check) as last_check'
' on s.host_object_id = c.host_object_id'
)
if not ido_tests.check_last_host_status_update(hostchecks_data):
- return 1
+ failures = True
servicechecks_data = run_query(
'select c2.alias, c1.display_name, unix_timestamp(s.last_check) as last_check'
' on c1.host_object_id = c2.host_object_id'
)
if not ido_tests.check_last_service_status_update(servicechecks_data):
- return 1
+ failures = True
logentry_info = run_query(
'select hosts.alias,'
' on hist.object_id = hosts.host_object_id and hist.state_type = 1'
)
if not ido_tests.check_logentries(logentry_info):
- return 1
+ failures = True
- return 0
+ return 1 if failures else 0
if __name__ == '__main__':
def main():
+ failures = False
+
run_query = lambda q: utils.run_pgsql_query(q, b'/usr/bin/psql')
if not ido_tests.validate_tables([d['Name'] for d in run_query('\\dt')
if d['Type'] == 'table']):
- return 1
+ return 1 # Bail out as we cannot proceed without any data
host_info = run_query('select * from icinga_hosts')
if not ido_tests.verify_host_config(host_info):
- return 1
+ return 1 # Bail out as we cannot proceed without any data
service_info = run_query(
'select c2.alias, c1.* from icinga_services as c1 '
' on c1.host_object_id = c2.host_object_id'
)
if not ido_tests.verify_service_config(service_info):
- return 1
+ return 1 # Bail out as we cannot proceed without any data
hostchecks_data = run_query(
'select c.alias, unix_timestamp(s.last_check) as last_check'
' on s.host_object_id = c.host_object_id'
)
if not ido_tests.check_last_host_status_update(hostchecks_data):
- return 1
+ failures = True
servicechecks_data = run_query(
'select c2.alias, c1.display_name, unix_timestamp(s.last_check) as last_check'
' on c1.host_object_id = c2.host_object_id'
)
if not ido_tests.check_last_service_status_update(servicechecks_data):
- return 1
+ failures = True
logentry_info = run_query(
'select hosts.alias,'
'group by hosts.alias'
)
if not ido_tests.check_logentries(logentry_info):
- return 1
+ failures = True
- return 0
+ return 1 if failures else 0
if __name__ == '__main__':
if [ ! -e /var/run/icinga2/cmd/livestatus ];
then
- echo "Icinga2 Livestatus socket not found"
+ echo "[FAIL] Icinga2 Livestatus socket not found"
exit 1
fi
fi
-echo "Icinga2 Livestatus socket found"
+echo "[OK] Icinga2 Livestatus socket found"
exit 0
if sudo test -f /var/log/icinga2/icinga2.log;
then
- echo "Icinga2 log file found"
+ echo "[OK] Icinga2 log file found"
exit 0
else
- echo "Icinga2 log file not found"
+ echo "[FAIL] Icinga2 log file not found"
exit 1
fi
#!/bin/sh
-sudo service mysqld status
+sudo service mysqld status &> /dev/null
+if [ $? -gt 0 ]; then
+ echo '[FAIL] mysqld is not running'
+ exit 1
+else
+ echo '[OK] mysqld is running'
+ exit 0
+fi
pid_after_restart=`ps -U icinga | grep icinga2 | awk '{print $1}'`
if [ $pid_after_restart -eq $pid_before_restart ]; then
- echo "Failed to send 'RESTART_PROCESS' to icinga2"
+ echo "[FAIL] Failed to send 'RESTART_PROCESS' to icinga2"
exit 1
else
- echo "Successfully sent 'RESTART_PROCESS' to icinga2"
+ echo "[OK] Successfully sent 'RESTART_PROCESS' to icinga2"
fi
printf "localhost\t0\tA passive result returning OK\n" | sudo send_nsca
if [ $? -gt 0 ]; then
- echo "Failed to send passive check result for host 'localhost'"
+ echo "[FAIL] Failed to send passive check result for host 'localhost'"
exit 1
else
- echo "Successfully sent a passive check result for host 'localhost'"
+ echo "[OK] Successfully sent a passive check result for host 'localhost'"
fi
printf "localhost\tdisk\t2\tA passive result not returning OK\n" | sudo send_nsca
if [ $? -gt 0 ]; then
- echo "Failed to send passive check result for service 'disk' on host 'localhost'"
+ echo "[FAIL] Failed to send passive check result for service 'disk' on host 'localhost'"
exit 1
else
- echo "Successfully sent a passive check result for service 'disk' on host 'localhost'"
+ echo "[OK] Successfully sent a passive check result for service 'disk' on host 'localhost'"
fi
#!/bin/sh
-sudo service postgresql status
+sudo service postgresql status &> /dev/null
+if [ $? -gt 0 ]; then
+ echo '[FAIL] postgresql is not running'
+ exit 1
+else
+ echo '[OK] postgresql is running'
+ exit 0
+fi
if [ -f /var/run/icinga2/icinga2.pid ];
then
- echo "Icinga2 pidfile found"
+ echo "[OK] Icinga2 pidfile found"
exit 0
else
- echo "Icinga2 pidfile not found"
+ echo "[FAIL] Icinga2 pidfile not found"
exit 1
fi
DEVNULL = open(os.devnull, 'w')
+class Logger(object):
+ INFO = 1
+ ERROR = 2
+ FAIL = 3
+ OK = 4
+
+ @staticmethod
+ def write(text, stderr=False):
+ if stderr:
+ sys.stderr.write(text)
+ sys.stderr.flush()
+ else:
+ sys.stdout.write(text)
+ sys.stdout.flush()
+
+ @classmethod
+ def log(cls, severity, text):
+ if severity == cls.INFO:
+ cls.write('\033[1;94m[INFO]\033[1;0m {0}'.format(text))
+ elif severity == cls.ERROR:
+ cls.write('\033[1;33m[ERROR]\033[1;0m {0}'.format(text), True)
+ elif severity == cls.FAIL:
+ cls.write('\033[1;31m[FAIL] {0}\033[1;0m'.format(text))
+ elif severity == cls.OK:
+ cls.write('\033[1;32m[OK]\033[1;0m {0}'.format(text))
+
+ @classmethod
+ def info(cls, text):
+ cls.log(cls.INFO, text)
+
+ @classmethod
+ def error(cls, text):
+ cls.log(cls.ERROR, text)
+
+ @classmethod
+ def fail(cls, text):
+ cls.log(cls.FAIL, text)
+
+ @classmethod
+ def ok(cls, text):
+ cls.log(cls.OK, text)
+
+
class TestSuite(object):
def __init__(self, configpath):
self._tests = []
def run(self):
for path in self._tests:
test_name = os.path.basename(path)
- self._apply_setup_routines(test_name, 'setup')
+ Logger.info('Copying test "{0}" to remote machine\n'.format(test_name))
self._copy_test(path)
- self._results[test_name] = self._run_test(path)
+ self._apply_setup_routines(test_name, 'setup')
+ Logger.info('Running test "{0}"...\n'.format(test_name))
+ result = self._run_test(path)
+ Logger.info('Test "{0}" has finished (Total tests: {1}, Failures: {2})\n'
+ ''.format(test_name, result['total'], result['failures']))
self._apply_setup_routines(test_name, 'teardown')
+ Logger.info('Removing test "{0}" from remote machine\n'.format(test_name))
self._remove_test(test_name)
+ self._results[test_name] = result
+ Logger.write('\n')
def _apply_setup_routines(self, test_name, context):
instructions = next((t[1].get(context)
for t in self._config.get('setups', {}).iteritems()
if re.match(t[0], test_name)), None)
if instructions is not None:
+ Logger.info('Applying {0} routines for test "{1}" .. '
+ ''.format(context, test_name))
for instruction in instructions.get('copy', []):
source, _, destination = instruction.partition('>>')
self._copy_file(source.strip(), destination.strip())
self._remove_file(filepath)
for command in instructions.get('exec', []):
self._exec_command(command)
+ Logger.write('Done\n')
def _remove_file(self, path):
command = self._config['commands']['clean'].format(path)
rc = subprocess.call(command, stdout=DEVNULL, shell=True)
if rc != 0:
- print 'WARNING: Cannot remove file "{0}" ({1})'.format(path, rc)
+ Logger.error('Cannot remove file "{0}" ({1})\n'.format(path, rc))
def _exec_command(self, command):
command = self._config['commands']['exec'].format(command)
rc = subprocess.call(command, stdout=DEVNULL, shell=True)
if rc != 0:
- print 'WARNING: Command "{0}" exited with exit code "{1}"' \
- ''.format(command, rc)
+ Logger.error('Command "{0}" exited with exit code "{1}"' \
+ ''.format(command, rc))
def _copy_file(self, source, destination):
command = self._config['commands']['copy'].format(source, destination)
rc = subprocess.call(command, stdout=DEVNULL, shell=True)
if rc != 0:
- print 'WARNING: Cannot copy file "{0}" to "{1}" ({2})' \
- ''.format(source, destination, rc)
+ Logger.error('Cannot copy file "{0}" to "{1}" ({2})' \
+ ''.format(source, destination, rc))
def _copy_test(self, path):
self._copy_file(path, os.path.join(self._config['settings']['test_root'],
os.path.basename(path))
p = subprocess.Popen(command.format(target), stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
- out, err = p.communicate()
-
+ output, test_count, failed_tests = self._watch_output(p.stdout)
return {
- 'stdout': out.decode('utf-8'),
- 'stderr': err.decode('utf-8'),
- 'returncode': p.returncode
+ 'total': test_count,
+ 'failures': failed_tests,
+ 'stdout': output,
+ 'stderr': p.stderr.read().decode('utf-8'),
+ 'returncode': p.wait()
}
+ def _watch_output(self, pipe):
+ output, total, failures = '', 0, 0
+ while True:
+ line = pipe.readline().decode('utf-8')
+ if not line:
+ break
+
+ if line.startswith('[ERROR] '):
+ Logger.error(line[8:])
+ elif line.startswith('[FAIL] '):
+ Logger.fail(line[7:])
+ failures += 1
+ total += 1
+ elif line.startswith('[OK] '):
+ Logger.ok(line[5:])
+ total += 1
+ else:
+ Logger.info(line.replace('[INFO] ', ''))
+
+ output += line
+ return (output, total, failures)
+
def parse_commandline():
- parser = OptionParser(version='0.1')
+ parser = OptionParser(version='0.2')
parser.add_option('-C', '--config', default="run_tests.conf",
help='The path to the config file to use [%default]')
- parser.add_option('-O', '--output',
- help='The file which to save the test results. '
- '(By default this goes to stdout)')
+ parser.add_option('-R', '--results',
+ help='The file where to store the test results')
return parser.parse_args()
suite.run()
- report = suite.get_report()
- if options.output is None:
- print report.encode('utf-8')
- else:
- with open(options.output, 'w') as f:
- f.write(report.encode('utf-8'))
+ if options.results is not None:
+ with open(options.results, 'w') as f:
+ f.write(suite.get_report().encode('utf-8'))
return 0
if [ $n -eq 3 ];
then
- echo "Icinga2 status.dat not found"
+ echo "[FAIL] Icinga2 status.dat not found"
exit 1
fi
fi
-echo "Icinga2 status.dat found"
+echo "[OK] Icinga2 status.dat found"
if [ -f /var/cache/icinga2/objects.cache ];
then
- echo "Icinga2 objects.cache found"
+ echo "[OK] Icinga2 objects.cache found"
else
- echo "Icinga2 objects.cache not found"
+ echo "[FAIL] Icinga2 objects.cache not found"
exit 1
fi
if [ $new_status_time -eq $status_time ];
then
- echo "Icinga2 status.dat is not being updated"
+ echo "[FAIL] Icinga2 status.dat is not being updated"
exit 1
else
- echo "Icinga2 status.dat is being updated"
+ echo "[OK] Icinga2 status.dat is being updated"
fi
projects / icinga2 / commitdiff