Commit 0e94ba48efb7771272fce14532b055bf440eaee1

Authored by John Snowdon
0 parents
Exists in master

first commit

Showing 4 changed files with 517 additions and 0 deletions   Show diff stats
data.py
... ... @@ -0,0 +1,281 @@
  1 +#!/usr/bin/env python
  2 +
  3 +import settings
  4 +import psutil
  5 +import os
  6 +import socket
  7 +import urllib
  8 +
  9 +class data():
  10 +
  11 + cpu_mhz = 0
  12 + cpu_percent = []
  13 + cpu_percent_avg = 0
  14 + network_media = False
  15 + network_status = False
  16 + network_address = "0.0.0.0"
  17 + disk_space = 0
  18 + disk_size = 0
  19 + nfs_space = 0
  20 + nfs_size = 0
  21 + nfs_mounted = False
  22 + load_average = (0.0, 0.0, 0.0)
  23 + ram_total = 0
  24 + ram_avail = 0
  25 + ram_buffer = 0
  26 + errors = []
  27 +
  28 + stats_before = {
  29 + 'net_bytes_read' : 0,
  30 + 'net_bytes_write' : 0,
  31 + 'disk_bytes_read' : 0,
  32 + 'disk_bytes_write' : 0,
  33 + }
  34 +
  35 + stats_now = {
  36 + 'net_bytes_read' : 0,
  37 + 'net_bytes_write' : 0,
  38 + 'disk_bytes_read' : 0,
  39 + 'disk_bytes_write' : 0,
  40 + }
  41 + stats_bandwidth = {
  42 + 'net_bytes_read_bw' : 0,
  43 + 'net_bytes_write_bw' : 0,
  44 + 'disk_bytes_read_bw' : 0,
  45 + 'disk_bytes_write_bw' : 0,
  46 + }
  47 +
  48 + def __init__(self):
  49 + """ Init """
  50 +
  51 + def zap(self):
  52 + """ Reset counters """
  53 + self.stats_before['net_bytes_read'] = 0
  54 + self.stats_before['net_bytes_write'] = 0
  55 + self.stats_before['disk_bytes_read'] = 0
  56 + self.stats_before['disk_bytes_write'] = 0
  57 +
  58 + self.stats_now['net_bytes_read'] = 0
  59 + self.stats_now['net_bytes_write'] = 0
  60 + self.stats_now['disk_bytes_read'] = 0
  61 + self.stats_now['disk_bytes_write'] = 0
  62 +
  63 + self.stats_bandwidth['net_bytes_read_bw'] = 0
  64 + self.stats_bandwidth['net_bytes_write_bw'] = 0
  65 + self.stats_bandwidth['disk_bytes_read_bw'] = 0
  66 + self.stats_bandwidth['disk_bytes_write_bw'] = 0
  67 +
  68 + self.nfs_size = 0
  69 + self.nfs_space = 0
  70 + self.errors = []
  71 +
  72 + def update(self, services = False):
  73 + """ Update values """
  74 +
  75 + self.network()
  76 + self.cpu()
  77 + self.disk()
  78 + self.memory()
  79 + self.bandwidth()
  80 +
  81 + if services:
  82 + self.services()
  83 +
  84 + def bandwidth(self):
  85 + """ Calculate bandwidth use """
  86 +
  87 + # Calculate used network bandwidth
  88 + self.stats_bandwidth["net_bytes_read_bw"] = (self.stats_now["net_bytes_read"] - self.stats_before["net_bytes_read"]) / settings.INTERVAL
  89 + self.stats_bandwidth["net_bytes_write_bw"] = (self.stats_now["net_bytes_write"] - self.stats_before["net_bytes_write"]) / settings.INTERVAL
  90 + # Get a total of read+write
  91 + self.stats_bandwidth["net_bytes_rw_bw"] = self.stats_bandwidth["net_bytes_read_bw"] + self.stats_bandwidth["net_bytes_write_bw"]
  92 +
  93 + # Calculate used disk bandwidth
  94 + self.stats_bandwidth["disk_bytes_read_bw"] = (self.stats_now["disk_bytes_read"] - self.stats_before["disk_bytes_read"]) / settings.INTERVAL
  95 + self.stats_bandwidth["disk_bytes_write_bw"] = (self.stats_now["disk_bytes_write"] - self.stats_before["disk_bytes_write"]) / settings.INTERVAL
  96 + # Get a total of read+write
  97 + self.stats_bandwidth["disk_bytes_rw_bw"] = self.stats_bandwidth["disk_bytes_read_bw"] + self.stats_bandwidth["disk_bytes_write_bw"]
  98 +
  99 + # Update counters for next loop
  100 + self.stats_before["net_bytes_read"] = self.stats_now["net_bytes_read"]
  101 + self.stats_before["net_bytes_write"] = self.stats_now["net_bytes_write"]
  102 + self.stats_before["disk_bytes_read"] = self.stats_now["disk_bytes_read"]
  103 + self.stats_before["disk_bytes_write"] = self.stats_now["disk_bytes_write"]
  104 +
  105 + def memory(self):
  106 + """ RAM information """
  107 +
  108 + # Get ram info
  109 + m = psutil.virtual_memory()
  110 +
  111 + self.ram_total = m.total
  112 + self.ram_avail = m.available
  113 + self.ram_buffer = m.buffers
  114 +
  115 + def network(self):
  116 + """ Network information """
  117 +
  118 + # Get network read/write byte count
  119 + n = psutil.net_io_counters()
  120 +
  121 + # Update counters
  122 + self.stats_now["net_bytes_write"] = n.bytes_sent
  123 + self.stats_now["net_bytes_read"] = n.bytes_recv
  124 +
  125 + # Get addresses currently in use
  126 + e = psutil.net_if_addrs()
  127 + try:
  128 + for v in e[settings.ETH_DEVICE]:
  129 + # Family=2 is the IPV4 details
  130 + if v.family == 2:
  131 + self.network_address = v.address
  132 + except Exception as e:
  133 + self.network_address = "0.0.0.0"
  134 +
  135 + # Get interface up/down status
  136 + try:
  137 + e = psutil.net_if_stats()
  138 + if e[settings.ETH_DEVICE].isup == True:
  139 + self.network_status = "Up"
  140 + else:
  141 + self.network_status = False
  142 + if settings.ERR_NETWORK_STATUS not in self.errors:
  143 + self.errors.append(settings.ERR_NETWORK_STATUS)
  144 + except Exception as e:
  145 + self.network_status = False
  146 + if settings.ERR_NETWORK_STATUS not in self.errors:
  147 + self.errors.append(settings.ERR_NETWORK_STATUS)
  148 +
  149 + # Get media connection status
  150 + try:
  151 + f = open("/sys/class/net/" + settings.ETH_DEVICE + "/carrier", "ro")
  152 + fdata = f.read().strip('\n')
  153 + f.close()
  154 + if fdata == '1':
  155 + self.network_media = "Ok"
  156 + else:
  157 + self.network_media = False
  158 + if settings.ERR_NETWORK_MEDIA not in self.errors:
  159 + self.errors.append(settings.ERR_NETWORK_MEDIA)
  160 + except Exception as e:
  161 + self.network_media = False
  162 + if settings.ERR_NETWORK_MEDIA not in self.errors:
  163 + self.errors.append(settings.ERR_NETWORK_MEDIA)
  164 +
  165 + def disk(self):
  166 + """ Disk information """
  167 +
  168 + # Get network read/write byte count
  169 + d = psutil.disk_io_counters()
  170 +
  171 + # Update counters
  172 + self.stats_now["disk_bytes_write"] = d.write_bytes
  173 + self.stats_now["disk_bytes_read"] = d.read_bytes
  174 +
  175 + # Get available disk space
  176 + try:
  177 + d = psutil.disk_usage(settings.DISK_PATH)
  178 + self.disk_space = d.free
  179 + self.disk_size = d.total
  180 + except Exception as e:
  181 + if settings.ERR_DISK_STATUS not in self.errors:
  182 + self.errors.append(settings.ERR_DISK_STATUS)
  183 + self.disk_space = 0
  184 + self.disk_size = 0
  185 +
  186 + # Get available nfs space and check it is mounted.
  187 + try:
  188 + d = psutil.disk_usage(settings.NFS_PATH)
  189 + if os.path.ismount(settings.NFS_PATH):
  190 + pass
  191 + else:
  192 + if settings.ERR_NFS_STATUS not in self.errors:
  193 + self.errors.append(settings.ERR_NFS_STATUS)
  194 + self.nfs_space = d.free
  195 + self.nfs_size = d.total
  196 + except Exception as e:
  197 + if settings.ERR_NFS_STATUS not in self.errors:
  198 + self.errors.append(settings.ERR_NFS_STATUS)
  199 + self.nfs_space = 0
  200 + self.nfs_size = 0
  201 +
  202 + def cpu(self):
  203 + """ Processor utilisation """
  204 + self.cpu_percent = psutil.cpu_percent(interval=None, percpu=True)
  205 + self.cpu_percent_avg = psutil.cpu_percent(interval=None, percpu=False)
  206 + self.load_average = os.getloadavg()
  207 +
  208 + f = open('/proc/cpuinfo', 'r')
  209 + cpuinfo = f.readlines()
  210 + f.close()
  211 + for l in cpuinfo:
  212 + if 'MHz' in l:
  213 + self.cpu_mhz = l.split(':')[1].split('.')[0].strip()
  214 +
  215 + def services(self):
  216 + """ Check status of services """
  217 +
  218 + # Check SSH
  219 + if "ssh" in settings.SERVICE_CHECKS:
  220 + try:
  221 + s = socket.create_connection((self.network_address, "22"))
  222 + s.close()
  223 + except Exception as e:
  224 + if settings.ERR_SERVICE_SSH not in self.errors:
  225 + self.errors.append(settings.ERR_SERVICE_SSH)
  226 +
  227 + # Check mysql
  228 + if "mysql" in settings.SERVICE_CHECKS:
  229 + try:
  230 + s = socket.create_connection((self.network_address, "3306"))
  231 + s.close()
  232 + except Exception as e:
  233 + if settings.ERR_SERVICE_MYSQL not in self.errors:
  234 + self.errors.append(settings.ERR_SERVICE_MYSQL)
  235 +
  236 + # check apache
  237 + if "httpd" in settings.SERVICE_CHECKS:
  238 + try:
  239 + u = urllib.urlopen(self.network_address)
  240 + if u.getcode() != 200:
  241 + if settings.ERR_SERVICE_HTTPDa not in self.errors:
  242 + self.errors.append(settings.ERR_SERVICE_HTTPDa)
  243 + except Exception as e:
  244 + if settings.ERR_SERVICE_HTTPDb not in self.errors:
  245 + self.errors.append(settings.ERR_SERVICE_HTTPDb)
  246 +
  247 + # check openstack keystone - identity
  248 + if "keystone" in settings.SERVICE_CHECKS:
  249 + if settings.SERVICE_CHECK_TYPE == "server":
  250 + # Check keystone server
  251 + pass
  252 +
  253 + # check openstack nova - compute
  254 + if "nova" in settings.SERVICE_CHECKS:
  255 + if settings.SERVICE_CHECK_TYPE == "server":
  256 + # Check nova server daemons
  257 + pass
  258 + else:
  259 + # Check nova client daemons
  260 + pass
  261 +
  262 + # check openstack cinder - block
  263 + if "cinder" in settings.SERVICE_CHECKS:
  264 + if settings.SERVICE_CHECK_TYPE == "server":
  265 + # Check cinder server
  266 + pass
  267 +
  268 + # check openstack glance - image/iso
  269 + if "glance" in settings.SERVICE_CHECKS:
  270 + if settings.SERVICE_CHECK_TYPE == "server":
  271 + # Check glance server
  272 + pass
  273 +
  274 + # check openstack neutron - networking
  275 + if "neutron" in settings.SERVICE_CHECKS:
  276 + if settings.SERVICE_CHECK_TYPE == "server":
  277 + # Check neutron server related daemons
  278 + pass
  279 + else:
  280 + # Check neutron client related daemons
  281 + pass
... ...
output.py
... ... @@ -0,0 +1,164 @@
  1 +#!/usr/bin/env python
  2 +
  3 +import time
  4 +
  5 +import settings
  6 +from data import data
  7 +
  8 +class Output():
  9 +
  10 + def __init__(self, mode = "terminal"):
  11 + self.mode = mode
  12 +
  13 + def sizeof_fmt(self, num, suffix='B'):
  14 + """ Handy convertor for unit suffixes """
  15 + for unit in ['','K','M','G','T','P','E','Z']:
  16 + if abs(num) < 1024.0:
  17 + return "%3.0f%s%s" % (num, unit, suffix)
  18 + num /= 1024.0
  19 + return "%.1f%s%s" % (num, 'Yi', suffix)
  20 +
  21 + def lcd_overviwe(self, data, interval):
  22 + """ Show the information on the lcd serial screen """
  23 + pass
  24 +
  25 + def terminal_overview(self, data, interval):
  26 + """ SHow the main overview screen on a unix console """
  27 +
  28 + # Display error conditions first and foremost
  29 + error = False
  30 + if len(data.errors) > 0:
  31 + # Loop over and scroll each error - ignore the normal cycle process in the outer call
  32 + e_num = 1
  33 + for e in data.errors:
  34 +
  35 + display_text = "%s: %s" % (e_num, e)
  36 +
  37 + if len(display_text) <= settings.DISPLAY_WIDTH:
  38 + # Display normal errors that fit in the width
  39 + print "%s Errors detected!" % len(data.errors)
  40 + print display_text
  41 +
  42 + else:
  43 + # Display longer errors that need to be scrolled
  44 + display_text = ""
  45 + e = "%s: %s" % (e_num, e)
  46 + for i in range (0, len(e)):
  47 + print "%s Errors detected!" % len(data.errors)
  48 + display_text = e[i:(i+settings.DISPLAY_WIDTH-1)]
  49 + print display_text
  50 + if i == 0:
  51 + # Pause the first time we show the beginning of the error.
  52 + time.sleep(1)
  53 + else:
  54 + # Slight gap between each character scrolling by.
  55 + time.sleep(settings.SCROLL_SPEED)
  56 +
  57 + # Sleep for next error
  58 + time.sleep(1)
  59 + e_num += 1
  60 + error = True
  61 +
  62 + # Don't bother printing metrics if we have errors to show!
  63 + if error:
  64 + return
  65 +
  66 + # Are we showing alternate screen 1?
  67 + # Show CPU
  68 + # Show IP address
  69 + if (interval >= settings.DETAIL_1_INTERVAL) and (interval <= (settings.DETAIL_1_INTERVAL + settings.DETAIL_DURATION)):
  70 + #print "CPU %2d%% %2d%% %2d%% %2d%%" % (data.cpu_percent[0], data.cpu_percent[1], data.cpu_percent[2], data.cpu_percent[3])
  71 + print "CPU %2d%% %sMHz" % (data.cpu_percent_avg, data.cpu_mhz)
  72 + print "IP %s" % (data.network_address)
  73 + return
  74 +
  75 + # Are we showing alternate screen 2?
  76 + # Show CPU
  77 + # Show CPU load
  78 + if (interval >= settings.DETAIL_2_INTERVAL) and (interval <= (settings.DETAIL_2_INTERVAL + settings.DETAIL_DURATION)):
  79 + #print "CPU %2d%% %2d%% %2d%% %2d%%" % (data.cpu_percent[0], data.cpu_percent[1], data.cpu_percent[2], data.cpu_percent[3])
  80 + print "CPU %2d%% %sMHz" % (data.cpu_percent_avg, data.cpu_mhz)
  81 + print "LOAD %s %s %s" % (data.load_average[0], data.load_average[1], data.load_average[2])
  82 + return
  83 +
  84 + # Are we showing alternate screen 3?
  85 + # Show CPU
  86 + # Show Disk space
  87 + if (interval >= settings.DETAIL_3_INTERVAL) and (interval <= (settings.DETAIL_3_INTERVAL + settings.DETAIL_DURATION)):
  88 + #print "CPU %2d%% %2d%% %2d%% %2d%%" % (data.cpu_percent[0], data.cpu_percent[1], data.cpu_percent[2], data.cpu_percent[3])
  89 + print "CPU %2d%% %sMHz" % (data.cpu_percent_avg, data.cpu_mhz)
  90 + print "DSK %s / %s" % (self.sizeof_fmt(num = data.disk_space), self.sizeof_fmt(num = data.disk_size))
  91 + return
  92 +
  93 + # Are we showing alternate screen 4?
  94 + # Show CPU
  95 + # Show RAM use
  96 + if (interval == settings.DETAIL_4_INTERVAL):
  97 + #print "CPU %2d%% %2d%% %2d%% %2d%%" % (data.cpu_percent[0], data.cpu_percent[1], data.cpu_percent[2], data.cpu_percent[3])
  98 + print "CPU %2d%% %sMHz" % (data.cpu_percent_avg, data.cpu_mhz)
  99 + display_text = ""
  100 + t = "RAM %s of %s used, %s buffers." % (self.sizeof_fmt(num = data.ram_avail), self.sizeof_fmt(num = data.ram_total), (self.sizeof_fmt(num = data.ram_buffer)))
  101 + for i in range (0, len(t)):
  102 + print "CPU %2d%% %sMHz" % (data.cpu_percent_avg, data.cpu_mhz)
  103 + display_text = t[i:(i+settings.DISPLAY_WIDTH-1)]
  104 + print display_text
  105 + if i == 0:
  106 + # Pause the first time we show the beginning of the error.
  107 + time.sleep(1)
  108 + else:
  109 + # Slight gap between each character scrolling by.
  110 + time.sleep(settings.SCROLL_SPEED)
  111 + return
  112 +
  113 + # Show the normal screen
  114 + # Show CPU
  115 + # Show total network and disk IO
  116 + #print "CPU %2d%% %2d%% %2d%% %2d%%" % (data.cpu_percent[0], data.cpu_percent[1], data.cpu_percent[2], data.cpu_percent[3])
  117 + print "CPU %2d%% %sMHz" % (data.cpu_percent_avg, data.cpu_mhz)
  118 + print "IO %s/s %s/s" % (self.sizeof_fmt(num = data.stats_bandwidth["net_bytes_rw_bw"]), self.sizeof_fmt(num = data.stats_bandwidth["disk_bytes_rw_bw"]))
  119 + return
  120 +
  121 + def run(self):
  122 + """ Main loop that calls the appropriate output module. """
  123 +
  124 + # Instantiate a new copy of the data acquisition class
  125 + d = data()
  126 + # Do the first collection of system metrics
  127 + d.update(services = True)
  128 + c = 0
  129 + interval = 0
  130 + while True:
  131 + c += 1
  132 + interval += 1
  133 +
  134 + # Collect system metrics
  135 + if interval == settings.SERVICE_INTERVAL:
  136 + # Service metrics take longer, so we only
  137 + # collected them at certain intervals
  138 + d.update(services = True)
  139 + else:
  140 + # Grab all the normal system metrics
  141 + d.update()
  142 +
  143 + # Call the correct output module
  144 + if self.mode == "terminal":
  145 + self.terminal_overview(d, interval)
  146 + elif self.mode == "lcd":
  147 + self.lcd_overview(d, interval)
  148 + else:
  149 + self.terminal_overview(d, interval)
  150 +
  151 + # Reset the display loop after showing the last alternate screen
  152 + if interval >= (settings.DETAIL_4_INTERVAL):
  153 + interval = 0
  154 +
  155 + # Wait until we're supposed to wake up next
  156 + time.sleep(settings.INTERVAL)
  157 +
  158 + # Zap counters to prevent overflows or
  159 + # memory growth after every N loops
  160 + if c == settings.ZAP_INTERVAL:
  161 + d.zap()
  162 + c = 0
  163 + d.update(services = True)
  164 +
... ...
run.py
... ... @@ -0,0 +1,6 @@
  1 +#!/usr/bin/env python
  2 +
  3 +from output import Output
  4 +
  5 +t = Output(mode="wibble")
  6 +t.run()
... ...
settings.py
... ... @@ -0,0 +1,66 @@
  1 +######################################
  2 +#
  3 +# Settings for the lcd_panel data
  4 +# gathering script.
  5 +#
  6 +######################################
  7 +
  8 +# How long we pause between data gathering
  9 +INTERVAL = 1
  10 +
  11 +# How long between service check intervals
  12 +SERVICE_INTERVAL = 10
  13 +
  14 +# How many loops before counters are reset
  15 +ZAP_INTERVAL = 50
  16 +
  17 +# How many rotations until the extended info detail screens are shown
  18 +DETAIL_1_INTERVAL = 10
  19 +DETAIL_2_INTERVAL = 20
  20 +DETAIL_3_INTERVAL = 30
  21 +DETAIL_4_INTERVAL = 40
  22 +# How many rotations each detail screen is shown before cycling back to the main screen
  23 +DETAIL_DURATION = 3
  24 +
  25 +# Which filesystem path should we check
  26 +# the disk useage of
  27 +DISK_PATH = "/var"
  28 +
  29 +# An NFS mounted directory that we should check. Set to '/' to bypass NFS check.
  30 +NFS_PATH = "/"
  31 +
  32 +# Name of the primary network adapter
  33 +ETH_DEVICE = "eth1"
  34 +
  35 +# Serial display width - how many characters we can display before scrolling right to left
  36 +DISPLAY_WIDTH = 19
  37 +
  38 +# Time to pause between scroll updates
  39 +SCROLL_SPEED = 0.25
  40 +
  41 +# When running service checks, some are against a 'client' service, others are against a 'server' service. This
  42 +# tells the class which type of check to use. (an openstack node can only be a server or client, not both)
  43 +SERVICE_CHECK_TYPE = "server"
  44 +#SERVICE_CHECK_TYPE = "client"
  45 +
  46 +# List of enabled service checks
  47 +SERVICE_CHECKS = ["ssh"]
  48 +#SERVICE_CHECKS = ["ssh", "mysql", "httpd", "keystone", "nova", "cinder", "glance", "neutron"]
  49 +
  50 +# Error messages
  51 +ERR_NETWORK_STATUS = "Network error, interface unconfigured."
  52 +ERR_NETWORK_MEDIA = "Network error, Ethernet cable disconnected."
  53 +ERR_DISK_STATUS = "Disk error, %s is not mounted." % DISK_PATH
  54 +ERR_NFS_STATUS = "NFS error, %s is not mounted or unavailable." % NFS_PATH
  55 +ERR_SERVICE_SSH = "OpenSSH error or service not running."
  56 +ERR_SERVICE_MYSQL = "MySQL error or service not running."
  57 +ERR_SERVICE_HTTPDa = "Apache httpd error, web page not available."
  58 +ERR_SERVICE_HTTPDb = "Apache httpd error, service not running."
  59 +ERR_SERVICE_KEYSTONEa = "Keystone error or service not running."
  60 +ERR_SERVICE_KEYSTONEb = "Keystone client error or unable to contact controller."
  61 +ERR_SERVICE_NOVAa = "Nova error or service not running."
  62 +ERR_SERVICE_NOVAb = "Nova client error or unable to contact controller."
  63 +ERR_SERVICE_CINDER = "Cinder error or service not running."
  64 +ERR_SERVICE_GLANCE = "Glance error or service not running."
  65 +ERR_SERVICE_NEUTRONa = "Neutron error or service not running."
  66 +ERR_SERVICE_NEUTRONb = "Neutron client error or unable to contact controller."
0 67 \ No newline at end of file
... ...