From: Sebastien GODARD Date: Sat, 18 Oct 2014 13:03:07 +0000 (+0200) Subject: Add irqstat command X-Git-Tag: v11.1.2~3 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=9dea99963be5f1c2ba384105e28219d9e3414c70;p=sysstat Add irqstat command Add new "irqstat" command (contributed by Lance Shelton from SanDisk/Fusion-io). This command monitors /proc/interrupts and is designed for NUMA systems with many processors. irqstat is currently a Python script that you can find in the contrib directory. Signed-off-by: Sebastien GODARD --- diff --git a/contrib/irqstat/README.md b/contrib/irqstat/README.md new file mode 100644 index 0000000..524a0b4 --- /dev/null +++ b/contrib/irqstat/README.md @@ -0,0 +1,7 @@ +irqstat +======= + +A better way to watch /proc/interrupts, designed for NUMA systems with many processors. + +Contributed by Lance Shelton + diff --git a/contrib/irqstat/irqstat b/contrib/irqstat/irqstat new file mode 100755 index 0000000..2efee7e --- /dev/null +++ b/contrib/irqstat/irqstat @@ -0,0 +1,286 @@ +#!/usr/bin/python + +""" +A better way to watch /proc/interrupts, especially on large NUMA machines with +so many CPUs that /proc/interrupts is wider than the screen. Press '0'-'9' +for node views, 't' for node totals +""" + +import sys +import tty +import termios +from time import sleep +import subprocess +from optparse import OptionParser +import thread +import threading + +KEYEVENT = threading.Event() + + +def gen_numa(): + """Generate NUMA info""" + cpunodes = {} + numacores = {} + out = subprocess.Popen('numactl --hardware | grep cpus', shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + errtxt = out.stderr.readline() + if errtxt: + print errtxt + '\r\n' + print "Is numactl installed?\r" + exit(1) + for line in out.stdout.readlines(): + arr = line.split() + if arr[0] == "node" and arr[2] == "cpus:" and len(arr) > 3: + node = arr[1] + numacores[node] = arr[3:] + for core in arr[3:]: + cpunodes[core] = node + return numacores, cpunodes + +# input character, passed between threads +INCHAR = '' + + +def wait_for_input(): + """Get a single character of input, validate""" + global INCHAR + + acceptable_keys = ['0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '0', 't'] + while True: + key = sys.stdin.read(1) + + # simple just to exit on any invalid input + if not key in acceptable_keys: + thread.interrupt_main() + + # set the new input and notify the main thread + INCHAR = key + KEYEVENT.set() + + +def filter_found(name, filter_list): + """Check if IRQ name matches anything in the filter list""" + for filt in filter_list: + if filt in name: + return True + return False + + +def display_itop(seconds, rowcnt, iterations, sort, totals, dispnode, zero, + filters): + """Main I/O loop""" + irqs = {} + cpunodes = {} + numacores = {} + loops = 0 + width = len('NODEXX') + + print ("interactive commands -- " + "t: view totals, 0-9: view node, any other key: quit") + + while True: + # Grab the new display type at a time when nothing is in flux + if KEYEVENT.isSet(): + KEYEVENT.clear() + dispnode = INCHAR if INCHAR in numacores.keys() else '-1' + + out = open('/proc/interrupts', 'r') + header = out.readline() + cpus = [] + for name in header.split(): + num = name[3:] + cpus.append(num) + + # Only query the numa information when something is missing. + # This is effectively the first time and when any disabled CPUs + # are enabled + if not num in cpunodes.keys(): + numacores, cpunodes = gen_numa() + + for line in out.readlines(): + vals = line.split() + irqnum = vals[0].rstrip(':') + + # Optionally exclude rows that are not an IRQ number + if totals is None: + try: + num = int(irqnum) + except ValueError: + continue + + irq = {} + irq['cpus'] = [int(x) for x in vals[1:len(cpus)+1]] + irq['oldcpus'] = (irqs[irqnum]['cpus'] if irqnum in irqs + else [0] * len(cpus)) + irq['name'] = ' '.join(vals[len(cpus)+1:]) + irq['oldsum'] = irqs[irqnum]['sum'] if irqnum in irqs else 0 + irq['sum'] = sum(irq['cpus']) + irq['num'] = irqnum + + for node in numacores.keys(): + oldkey = 'oldsum' + node + key = 'sum' + node + irq[oldkey] = (irqs[irqnum][key] if irqnum in irqs + and key in irqs[irqnum] else 0) + irq[key] = 0 + + for idx, val in enumerate(irq['cpus']): + key = 'sum' + cpunodes[cpus[idx]] + irq[key] = irq[key] + val if key in irq else val + + # save old + irqs[irqnum] = irq + + def sort_func(val): + """Sort output""" + sortnum = -1 + try: + sortnum = int(sort) + except ValueError: + pass + + if sortnum >= 0: + for node in numacores.keys(): + if sortnum == int(node): + return val['sum' + node] - val['oldsum' + node] + if sort == 't': + return val['sum'] - val['oldsum'] + if sort == 'i': + return int(val['num']) + if sort == 'n': + return val['name'] + # reverse sort all IRQ count sorts + rev = sort not in ['i', 'n'] + rows = sorted(irqs.values(), key=sort_func, reverse=rev) + + # determine the width required for the count field + for idx, irq in enumerate(rows): + width = max(width, len(str(irq['sum'] - irq['oldsum']))) + + print "" + '\r' + print "IRQs / " + str(seconds) + " second(s)" + '\r' + fmtstr = ('IRQ# %' + str(width) + 's') % 'TOTAL' + + # node view header + if int(dispnode) >= 0: + node = 'NODE%s' % dispnode + fmtstr += (' %' + str(width) + 's ') % node + for idx, val in enumerate(irq['cpus']): + if cpunodes[cpus[idx]] == dispnode: + cpu = 'CPU%s' % cpus[idx] + fmtstr += (' %' + str(width) + 's ') % cpu + # top view header + else: + for node in sorted(numacores.keys()): + node = 'NODE%s' % node + fmtstr += (' %' + str(width) + 's ') % node + + fmtstr += ' NAME' + print fmtstr + '\r' + + displayed_rows = 0 + for idx, irq in enumerate(rows): + if len(filters) and not filter_found(irq['name'], filters): + continue + + total = irq['sum'] - irq['oldsum'] + if zero and not total: + continue + + # IRQ# TOTAL + fmtstr = ('%4s %' + str(width) + 'd') % (irq['num'], total) + + # node view + if int(dispnode) >= 0: + oldnodesum = 'oldsum' + dispnode + nodesum = 'sum' + dispnode + nodecnt = irq[nodesum] - irq[oldnodesum] + if zero and not nodecnt: + continue + fmtstr += (' %' + str(width) + 's ') % str(nodecnt) + for cpu, val in enumerate(irq['cpus']): + if cpunodes[cpus[cpu]] == dispnode: + fmtstr += ((' %' + str(width) + 's ') % + str(irq['cpus'][cpu] - irq['oldcpus'][cpu])) + + # top view + else: + for node in sorted(numacores.keys()): + oldnodesum = 'oldsum' + node + nodesum = 'sum' + node + nodecnt = irq[nodesum] - irq[oldnodesum] + fmtstr += ((' %' + str(width) + 's ') % str(nodecnt)) + fmtstr += ' ' + irq['name'] + print fmtstr + '\r' + displayed_rows += 1 + if displayed_rows == rowcnt: + break + + # Update field widths after the first iteration. Data changes + # significantly between the all-time stats and the interval stats, so + # this compresses the fields quite a bit. Updating every iteration + # is too jumpy. + if loops == 0: + width = len('NODEXX') + + loops += 1 + if loops == iterations: + break + + # thread.interrupt_main() does not seem to interrupt a sleep, so break + # it into tenth-of-a-second sleeps to improve user response time on exit + for _ in range(0, seconds * 10): + sleep(.1) + + +def main(args): + """Parse arguments, call main loop""" + + parser = OptionParser(description=__doc__) + parser.add_option("-i", "--iterations", default='-1', + help="iterations to run") + parser.add_option("-n", "--node", default='-1', + help="view a single node") + parser.add_option("-r", "--rows", default='10', + help="rows to display (default 10)") + parser.add_option("-s", "--sort", default='t', + help="column to sort on ('t':total, 'n': name, " + "'i':IRQ number, '1':node1, etc) (default: 't')") + parser.add_option("-t", "--time", default='5', + help="update interval in seconds") + parser.add_option("-z", "--zero", action="store_true", + help="exclude inactive IRQs") + parser.add_option("--filter", default="", + help="filter IRQs based on name matching comma " + "separated filters") + parser.add_option("--totals", action="store_true", + help="include total rows") + + options = parser.parse_args(args)[0] + + if options.filter: + options.filter = options.filter.split(',') + else: + options.filter = [] + + # Set the terminal to unbuffered, to catch a single keypress + out = sys.stdin.fileno() + old_settings = termios.tcgetattr(out) + tty.setraw(sys.stdin.fileno()) + + # input thread + thread.start_new_thread(wait_for_input, tuple()) + + try: + display_itop(int(options.time), int(options.rows), + int(options.iterations), options.sort, options.totals, + options.node, options.zero, options.filter) + except (KeyboardInterrupt, SystemExit): + pass + finally: + termios.tcsetattr(out, termios.TCSADRAIN, old_settings) + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/contrib/irqstat/pylint.rc b/contrib/irqstat/pylint.rc new file mode 100644 index 0000000..b0ab11f --- /dev/null +++ b/contrib/irqstat/pylint.rc @@ -0,0 +1,9 @@ +[MESSAGES CONTROL] +disable=too-many-arguments, + too-many-locals, + too-many-branches, + too-many-statements, + global-statement + +[REPORTS] +reports=no