from time import sleep import datetime import sys import json import os import promMon import mon_pkg_update #===INIT======================================================================== filedir = os.path.dirname(os.path.abspath(sys.argv[0])) vols_to_scan = {} vols_to_scan["zfs"] = [] vols_to_scan["non_zfs"] = [] nics_to_scan = [] SMART_data_update_period_seconds = 60 last_SMART_read_timestamp = 0 packages_data_update_period_seconds = 3600 last_packages_read_timestamp = 0 docker_data_update_period_seconds = 3600 last_docker_read_timestamp = 0 update_scan_period_seconds = 300 last_update_scan_timestamp = 0 update_from_git_tags = True gpu_vendor = "nVidia" # load config.json file try: with open(filedir+"/config.json", "r") as f: config = json.loads(f.read()) for module in config["modules"]: if "ryzenPower" in module: import ryzenPower if "kvmSensors" in module: import kvmSensors as ks if "sysinfo" in module: import sysinfo as si if "docker" in module: import docker as do if "SMART" in module: import SMART as sm if "sensors" in module: import sensors as lmsn if "intelPower" in module: import intelPower if "procMon" in module: import procMon as pr if "RAS" in module: import RAS if "cpuinfo" in module: import cpuinfo as cinfo if "gpu" in module: import gpu if "packages" in module: import packages try: if len(config["volumes"]["zfs"]) != 0: for i in config["volumes"]["zfs"]: vols_to_scan["zfs"].append(i) if len(config["volumes"]["non_zfs"]) != 0: for i in config["volumes"]["non_zfs"]: vols_to_scan["non_zfs"].append(i) except: print("No volumes object detected") try: if len(config["network"]["nics"]) != 0: for nic in config["network"]["nics"]: nics_to_scan.append(nic) except: print("No network object detected") try: gpu_vendor = config["gpu"] except: pass except: print("No config file found, exitting") exit(255) proc_last = 0 if "promMon" not in sys.modules: print("Prometheus exporter helper not inported, exitting") exit(1) pm = promMon.prometheus(port=9339, name="hw-monitor") pm.add_monitor("self_loop_time", "us") pm.add_monitor("version", "", tags=("version",)) if "docker" in sys.modules: pm.add_monitor("docker_version", "") pm.add_monitor("docker_overall_info", "", tags=("module","type")) pm.add_monitor("docker_overall_size", "B", tags=("module","type")) pm.add_monitor("docker_images", "", tags=("id","containers","created","repository","unique_size")) pm.add_monitor("docker_containers", "", tags=("id","created","image","volumes","mounts","name","networks","runtime","state","status")) pm.add_monitor("docker_volumes", "", tags=("name","driver","links","mountpoint")) pm.add_monitor("docker_build_cache", "", tags=("id","type","created","in_use","last_use","shared","use_count")) if "ryzenPower" in sys.modules: pm.add_monitor("package_power", "W") pm.add_monitor("core_total", "W") pm.add_monitor("core", "W") rp = ryzenPower.RyzenPower() if "intelPower" in sys.modules: ip = intelPower.IntelPower() if "kvmSensors" in sys.modules: pm.add_monitor("fan_rpm", "RPM") pm.add_monitor("fan_ok", "") pm.add_monitor("temp_celsius", "C") pm.add_monitor("temp_ok", "") pm.add_monitor("voltage", "V") pm.add_monitor("voltage_ok", "") if "cpuinfo" in sys.modules: pm.add_monitor("cpu_info", "", tags=("vendor","model","cpus")) if "sysinfo" in sys.modules: pm.add_monitor("disk_read", "B/s") pm.add_monitor("disk_write", "B/s") pm.add_monitor("disk_io_read", "iops") pm.add_monitor("disk_io_write", "iops") pm.add_monitor("disk_io_read_time", "ms") pm.add_monitor("disk_io_write_time", "ms") pm.add_monitor("disk_io_read_merged", "") pm.add_monitor("disk_io_write_merged", "") pm.add_monitor("disk_busy", "ms") pm.add_monitor("cpu_count", "") pm.add_monitor("cpu_frequency", "Hz") pm.add_monitor("cpu_usage", "%") pm.add_monitor("uptime", "s") pm.add_monitor("system_info", "", tags=("hostname","kernel","board")) pm.add_monitor("ip_addrs", "", tags=("interface","ip",)) pm.add_monitor("user_sessions", "") pm.add_monitor("users", "", tags=("user","from")) if "SMART" in sys.modules: pm.add_monitor("smart_attributes", "", tags=("serial","device","attribute","id","value","thres","worst","raw")) if "gpu" in sys.modules: pm.add_monitor("gpu_info", "", tags=("name","vbios","driver","pstate","uuid")) pm.add_monitor("gpu_util", "%", tags=("name","uuid","stat")) pm.add_monitor("gpu_throttle", "", tags=("name","uuid","stat")) pm.add_monitor("gpu_memory_used", "MB", tags=("name","uuid")) pm.add_monitor("gpu_memory_total", "MB", tags=("name","uuid")) pm.add_monitor("gpu_power", "W", tags=("name","uuid")) pm.add_monitor("gpu_power_limit", "W", tags=("name","uuid")) pm.add_monitor("gpu_temp", "C", tags=("name","uuid")) pm.add_monitor("gpu_fan_speed", "%", tags=("name","uuid")) pm.add_monitor("gpu_gpu_clock", "Hz", tags=("name","uuid")) pm.add_monitor("gpu_mem_clock", "Hz", tags=("name","uuid")) pm.add_monitor("gpu_sm_clock", "Hz", tags=("name","uuid")) pm.add_monitor("gpu_video_clock", "Hz", tags=("name","uuid")) pm.add_monitor("gpu_gpu_clock_max", "Hz", tags=("name","uuid")) pm.add_monitor("gpu_mem_clock_max", "Hz", tags=("name","uuid")) if "packages" in sys.modules: pm.add_monitor("update_pkg_count", "", tags=("package_mgr",)) pm.add_monitor("update_pkg_updatable", "", tags=("package","version","repository")) pm.add_monitor("installed_pkg_count", "", tags=("package_mgr",)) pm.add_monitor("installed_pkgs", "", tags=("package","version","repository")) pm.add_monitor("installed_pkg_size", "", tags=("package_mgr",)) def self_monitoring(name, start): end = datetime.datetime.now() task_time = end - start pm.monitor("self_loop_time", (name,), task_time.microseconds + (task_time.seconds * 1000000)) return datetime.datetime.now() while True: mon_time = 0 loop_start = datetime.datetime.now() #===VARIABLE INIT=============================================================== if "ryzenPower" in sys.modules: package, cores = rp.measure_nonblocking() mon_time = self_monitoring("zenpower", loop_start if mon_time == 0 else mon_time) if "kvmSensors" in sys.modules: kvm = ks.readSensors() mon_time = self_monitoring("kvm", loop_start if mon_time == 0 else mon_time) if "sysinfo" in sys.modules: disks = si.getDisk() mon_time = self_monitoring("disks", loop_start if mon_time == 0 else mon_time) cpu = si.getCPU() mon_time = self_monitoring("cpu", loop_start if mon_time == 0 else mon_time) memory = si.getMemory() mon_time = self_monitoring("memory", loop_start if mon_time == 0 else mon_time) try: partitions = si.getPartitions() mon_time = self_monitoring("partitions", loop_start if mon_time == 0 else mon_time) except: partitions = 0 try: zfs = si.getZFS() mon_time = self_monitoring("zfs", loop_start if mon_time == 0 else mon_time) except: zfs = 0 uptime = si.getUptime() mon_time = self_monitoring("uptime", loop_start if mon_time == 0 else mon_time) network = si.getNetwork() ipaddrs = si.getIP() mon_time = self_monitoring("network", loop_start if mon_time == 0 else mon_time) systeminfo = si.getSysInfo() mon_time = self_monitoring("systeminfo", loop_start if mon_time == 0 else mon_time) users = si.users() mon_time = self_monitoring("users", loop_start if mon_time == 0 else mon_time) if "SMART" in sys.modules: if last_SMART_read_timestamp == 0: smart = sm.getAllDeviceAttributes() last_SMART_read_timestamp = datetime.datetime.now() else: timedelta = (datetime.datetime.now() - last_SMART_read_timestamp).total_seconds() if timedelta >= SMART_data_update_period_seconds: smart = sm.getAllDeviceAttributes() last_SMART_read_timestamp = datetime.datetime.now() mon_time = self_monitoring("smart", loop_start if mon_time == 0 else mon_time) if "docker" in sys.modules: containers = do.health() try: if last_docker_read_timestamp == 0: docker_info = do.getInfo() docker_size = do.getSize() last_docker_read_timestamp = datetime.datetime.now() else: timedelta = (datetime.datetime.now() - last_docker_read_timestamp).total_seconds() if timedelta >= docker_data_update_period_seconds: docker_info = do.getInfo() docker_size = do.getSize() last_docker_read_timestamp = datetime.datetime.now() except Exception as e: print(e) mon_time = self_monitoring("containers", loop_start if mon_time == 0 else mon_time) if "sensors" in sys.modules: temperatures = lmsn.getTemps() mon_time = self_monitoring("temperatures", loop_start if mon_time == 0 else mon_time) fans = lmsn.getFans() mon_time = self_monitoring("fans", loop_start if mon_time == 0 else mon_time) voltages = lmsn.getVoltages() mon_time = self_monitoring("voltage", loop_start if mon_time == 0 else mon_time) if "intelPower" in sys.modules: package, cores = ip.measure_nonblocking() mon_time = self_monitoring("cpu_power", loop_start if mon_time == 0 else mon_time) if "RAS" in sys.modules: ras = RAS.readRAS() mon_time = self_monitoring("ras", loop_start if mon_time == 0 else mon_time) if "procMon" in sys.modules: proc = pr.exportProcesses(proc_last) proc_last = proc mon_time = self_monitoring("proc", loop_start if mon_time == 0 else mon_time) if "cpuinfo" in sys.modules: cpuinfo = cinfo.getCpuInfo() mon_time = self_monitoring("cpuinfo", loop_start if mon_time == 0 else mon_time) if "gpu" in sys.modules: gpuinfo = gpu.readGpu(gpu_vendor) mon_time = self_monitoring("gpu", loop_start if mon_time == 0 else mon_time) if "packages" in sys.modules: if last_packages_read_timestamp == 0: up = packages.getPackages() last_packages_read_timestamp = datetime.datetime.now() else: timedelta = (datetime.datetime.now() - last_packages_read_timestamp).total_seconds() if timedelta >= packages_data_update_period_seconds: up = packages.getPackages() last_packages_read_timestamp = datetime.datetime.now() mon_time = self_monitoring("packages", loop_start if mon_time == 0 else mon_time) if last_update_scan_timestamp == 0: version = mon_pkg_update.getCurrentTag() if update_from_git_tags: mon_pkg_update.update() last_update_scan_timestamp = datetime.datetime.now() else: timedelta = (datetime.datetime.now() - last_update_scan_timestamp).total_seconds() if timedelta >= update_scan_period_seconds: version = mon_pkg_update.getCurrentTag() if update_from_git_tags: mon_pkg_update.update() last_update_scan_timestamp = datetime.datetime.now() mon_time = self_monitoring("version_check", loop_start if mon_time == 0 else mon_time) #===MOVE VARS TO PROMETHEUS EXPORTER============================================ pm.monitor("version", (version,), 1) if "sysinfo" in sys.modules: pm.monitor("uptime", ("sensors",), uptime) pm.monitor("cpu_count", ("sensors",), cpu["cpu_count"]) pm.monitor("cpu_usage", ("sensors",), cpu["usage"]) for disk in disks: pm.monitor("disk_read", (disk,), disks[disk]["read"]) pm.monitor("disk_write", (disk,), disks[disk]["write"]) pm.monitor("disk_io_read", (disk,), disks[disk]["io_read"]) pm.monitor("disk_io_write", (disk,), disks[disk]["io_write"]) pm.monitor("disk_io_read_time", (disk,), disks[disk]["io_read_time"]) pm.monitor("disk_io_write_time", (disk,), disks[disk]["io_write_time"]) pm.monitor("disk_io_read_merged", (disk,), disks[disk]["io_read_merged"]) pm.monitor("disk_io_write_merged", (disk,), disks[disk]["io_write_merged"]) pm.monitor("disk_busy", (disk,), disks[disk]["busy"]) for core in cpu["frequency"]: pm.monitor("cpu_frequency", (str(core),), cpu["frequency"][core]) for t in cpu["time_percent"]: try: pm.monitor("cpu_time_percent", (t,), cpu["time_percent"][t]) except: pm.add_monitor("cpu_time_percent", "%") pm.monitor("cpu_time_percent", (t,), cpu["time_percent"][t]) for i in memory: try: pm.monitor("memory_"+i, ("memory",), memory[i]) except: pm.add_monitor("memory_"+i, "%") pm.monitor("memory_"+i, ("memory",), memory[i]) if zfs != 0: for pool in zfs: if len(vols_to_scan["zfs"]) != 0: if pool not in vols_to_scan["zfs"]: continue try: pm.monitor("zfs_state", (pool,), zfs[pool]["state"]) pm.monitor("zfs_size", (pool,), zfs[pool]["size"]) pm.monitor("zfs_used", (pool,), zfs[pool]["used"]) pm.monitor("zfs_free", (pool,), zfs[pool]["free"]) pm.monitor("zfs_fragmentation", (pool,), zfs[pool]["fragmentation"]) pm.monitor("zfs_dedup", (pool,), zfs[pool]["dedup"]) except: pm.add_monitor("zfs_state", "") pm.add_monitor("zfs_size", "B") pm.add_monitor("zfs_used", "B") pm.add_monitor("zfs_free", "B") pm.add_monitor("zfs_fragmentation", "%") pm.add_monitor("zfs_dedup", "") pm.monitor("zfs_state", (pool,), zfs[pool]["state"]) pm.monitor("zfs_size", (pool,), zfs[pool]["size"]) pm.monitor("zfs_used", (pool,), zfs[pool]["used"]) pm.monitor("zfs_free", (pool,), zfs[pool]["free"]) pm.monitor("zfs_fragmentation", (pool,), zfs[pool]["fragmentation"]) pm.monitor("zfs_dedup", (pool,), zfs[pool]["dedup"]) for nic in network: if len(nics_to_scan) != 0: if nic not in nics_to_scan: continue try: pm.monitor("network_rx", (nic,), network[nic]["rx"]) pm.monitor("network_tx", (nic,), network[nic]["tx"]) pm.monitor("network_err_rx", (nic,), network[nic]["err_rx"]) pm.monitor("network_err_tx", (nic,), network[nic]["err_tx"]) pm.monitor("network_drop_rx", (nic,), network[nic]["drop_rx"]) pm.monitor("network_drop_tx", (nic,), network[nic]["drop_tx"]) pm.monitor("network_packet_rx", (nic,), network[nic]["packet_rx"]) pm.monitor("network_packet_tx", (nic,), network[nic]["packet_tx"]) except: pm.add_monitor("network_rx", "B") pm.add_monitor("network_tx", "B") pm.add_monitor("network_err_rx", "") pm.add_monitor("network_err_tx", "") pm.add_monitor("network_drop_rx", "") pm.add_monitor("network_drop_tx", "") pm.add_monitor("network_packet_rx", "") pm.add_monitor("network_packet_tx", "") pm.monitor("network_rx", (nic,), network[nic]["rx"]) pm.monitor("network_tx", (nic,), network[nic]["tx"]) pm.monitor("network_err_rx", (nic,), network[nic]["err_rx"]) pm.monitor("network_err_tx", (nic,), network[nic]["err_tx"]) pm.monitor("network_drop_rx", (nic,), network[nic]["drop_rx"]) pm.monitor("network_drop_tx", (nic,), network[nic]["drop_tx"]) pm.monitor("network_packet_rx", (nic,), network[nic]["packet_rx"]) pm.monitor("network_packet_tx", (nic,), network[nic]["packet_tx"]) if partitions != 0: for part in partitions: if len(vols_to_scan["non_zfs"]) != 0: if part not in vols_to_scan["non_zfs"]: continue try: pm.monitor("partition_size", (part,), partitions[part]["size"]) pm.monitor("partition_used", (part,), partitions[part]["used"]) pm.monitor("partition_free", (part,), partitions[part]["free"]) except: pm.add_monitor("partition_size", "B") pm.add_monitor("partition_used", "B") pm.add_monitor("partition_free", "B") pm.monitor("partition_size", (part,), partitions[part]["size"]) pm.monitor("partition_used", (part,), partitions[part]["used"]) pm.monitor("partition_free", (part,), partitions[part]["free"]) pm.monitor("system_info", (systeminfo["hostname"],systeminfo["kernel"],systeminfo["board"]), 1) pm.delete_monitor("user_sessions") pm.delete_monitor("users") try: for user in users: pm.monitor("user_sessions", (user,), users[user]["sessions"]) for session in users[user]["session"]: pm.monitor("users", (user,users[user]["session"][session]["from"]), 1) except: pass pm.delete_monitor("ip_addrs") for interface in ipaddrs: pm.monitor("ip_addrs", (interface,ipaddrs[interface]), 1) if "ryzenPower" in sys.modules: pm.monitor("package_power", ("sensors",), package) core_total = 0 for core in cores: pm.monitor("core", (str(int(core/2)),), cores[core]) core_total += cores[core] pm.monitor("core_total", ("sensors",), core_total) if "kvmSensors" in sys.modules: for fan in kvm["fans"]: pm.monitor("fan_rpm", (kvm["fans"][fan]["id"],), kvm["fans"][fan]["rpm"]) if kvm["fans"][fan]["status"] == "ok": pm.monitor("fan_ok", (kvm["fans"][fan]["id"],), 1) else: pm.monitor("fan_ok", (kvm["fans"][fan]["id"],), 0) for temp in kvm["temp"]: pm.monitor("temp_celsius", (kvm["temp"][temp]["id"],), kvm["temp"][temp]["temp"]) if kvm["temp"][temp]["status"] == "ok": pm.monitor("temp_ok", (kvm["temp"][temp]["id"],), 1) else: pm.monitor("temp_ok", (kvm["temp"][temp]["id"],), 0) for volt in kvm["volt"]: pm.monitor("voltage", (kvm["volt"][volt]["id"],), kvm["volt"][volt]["voltage"]) if kvm["volt"][volt]["status"] == "ok": pm.monitor("voltage_ok", (kvm["volt"][volt]["id"],), 1) else: pm.monitor("voltage_ok", (kvm["volt"][volt]["id"],), 0) if "docker" in sys.modules: pm.delete_monitor("docker_version") pm.delete_monitor("docker_images") pm.delete_monitor("docker_containers") pm.delete_monitor("docker_volumes") pm.delete_monitor("docker_build_cache") for container in containers: try: pm.monitor("docker_status", (container,), containers[container]["status"]) pm.monitor("docker_health", (container,), containers[container]["health"]) except: pm.add_monitor("docker_status", "") pm.add_monitor("docker_health", "") pm.monitor("docker_status", (container,), containers[container]["status"]) pm.monitor("docker_health", (container,), containers[container]["health"]) try: pm.monitor("docker_version", (docker_info["version"],), 1) for module in docker_size: pm.monitor("docker_overall_info", (module,"count_total"), int(docker_size[module]["count_total"])) pm.monitor("docker_overall_info", (module,"count_active"), int(docker_size[module]["count_active"])) pm.monitor("docker_overall_size", (module,"used"), docker_size[module]["used"]) pm.monitor("docker_overall_size", (module,"reclaimable"), docker_size[module]["reclaimable"]) for image in docker_info["images"]: pm.monitor("docker_images", (image,docker_info["images"][image]["containers"],docker_info["images"][image]["created"],docker_info["images"][image]["repository"],docker_info["images"][image]["unique_size"]), docker_info["images"][image]["size"]) for container in docker_info["containers"]: pm.monitor("docker_containers", (container,docker_info["containers"][container]["created"],docker_info["containers"][container]["image"],docker_info["containers"][container]["volumes"],docker_info["containers"][container]["mounts"],docker_info["containers"][container]["name"],docker_info["containers"][container]["networks"],docker_info["containers"][container]["runtime"],docker_info["containers"][container]["state"],docker_info["containers"][container]["status"]), docker_info["containers"][container]["size"]) for volume in docker_info["volumes"]: pm.monitor("docker_volumes", (volume,docker_info["volumes"][volume]["driver"],docker_info["volumes"][volume]["links"],docker_info["volumes"][volume]["mountpoint"]), docker_info["volumes"][volume]["size"]) for cache in docker_info["buildcache"]: pm.monitor("docker_build_cache", (cache,docker_info["buildcache"][cache]["type"],docker_info["buildcache"][cache]["created"],docker_info["buildcache"][cache]["in_use"],docker_info["buildcache"][cache]["last_use"],docker_info["buildcache"][cache]["shared"],docker_info["buildcache"][cache]["use_count"]), docker_info["buildcache"][cache]["size"]) except Exception as e: print(e) if "sensors" in sys.modules: for temp in temperatures: try: for sensor in temperatures[temp]: if "coretemp" in temp: pm.monitor("temp_celsius_lm", (str(sensor),), temperatures[temp][sensor]) else: pm.monitor("temp_celsius_lm", (str(temp)+'_'+str(sensor),), temperatures[temp][sensor]) except: pm.add_monitor("temp_celsius_lm", "C") for sensor in temperatures[temp]: if "coretemp" in temp: pm.monitor("temp_celsius_lm", (str(sensor),), temperatures[temp][sensor]) else: pm.monitor("temp_celsius_lm", (str(temp)+'_'+str(sensor),), temperatures[temp][sensor]) for fan in fans: try: for sensor in fans[fan]: pm.monitor("fans_lm", (str(sensor),), fans[fan][sensor]) except: pm.add_monitor("fans_lm", "RPM") for sensor in fans[fan]: pm.monitor("fans_lm", (str(sensor),), fans[fan][sensor]) for voltage in voltages: try: for sensor in voltages[voltage]: pm.monitor("voltages_lm", (str(sensor),), voltages[voltage][sensor]) except: pm.add_monitor("voltages_lm", "V") for sensor in voltages[voltage]: pm.monitor("voltages_lm", (str(sensor),), voltages[voltage][sensor]) if "intelPower" in sys.modules: try: pm.monitor("intel_cpu_power", ("core",), cores) pm.monitor("intel_cpu_power", ("package",), package) except: pm.add_monitor("intel_cpu_power", "W") pm.monitor("intel_cpu_power", ("core",), cores) pm.monitor("intel_cpu_power", ("package",), package) if "RAS" in sys.modules: for item in ras: try: for tag in ras[item]: if tag == "total_errors": pm.monitor("ras_total", (item.replace(' ','_'),), ras[item][tag]) else: for error in ras[item][tag]: pm.monitor("ras_"+error, (item.replace(' ','_')+"_"+tag,), ras[item][tag][error]) except: for tag in ras[item]: if tag == "total_errors": try: pm.add_monitor("ras_total", "") pm.monitor("ras_total", (item.replace(' ','_'),), ras[item][tag]) except: pass else: for error in ras[item][tag]: try: pm.add_monitor("ras_"+error, "") pm.monitor("ras_"+error, (item.replace(' ','_')+"_"+tag,), ras[item][tag][error]) except: pass if "SMART" in sys.modules: try: pm.add_monitor("smart_bytes_written", "B", tags=("serial","instance")) except: pass pm.delete_monitor("smart_attributes") for device in smart: serial = smart[device]["serial_number"] pm.monitor("smart_bytes_written", (serial,device), smart[device]["bytes_written"]) for attr in smart[device]["data"]: if smart[device]["type"] == "ATA": try: pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr]["raw"]) pm.monitor("smart_value_"+attr, (serial,device), smart[device]["data"][attr]["value"]) pm.monitor("smart_thr_"+attr, (serial,device), smart[device]["data"][attr]["thr"]) pm.monitor("smart_worst_"+attr, (serial,device), smart[device]["data"][attr]["worst"]) pm.monitor("smart_attributes", (serial,device,attr,smart[device]["data"][attr]["id"],smart[device]["data"][attr]["value"],smart[device]["data"][attr]["thr"],smart[device]["data"][attr]["worst"],smart[device]["data"][attr]["raw"]), smart[device]["data"][attr]["raw"]) except: try: pm.add_monitor("smart_raw_"+attr, "", tags=("serial","instance")) pm.add_monitor("smart_value_"+attr, "", tags=("serial","instance")) pm.add_monitor("smart_thr_"+attr, "", tags=("serial","instance")) pm.add_monitor("smart_worst_"+attr, "", tags=("serial","instance")) pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr]["raw"]) pm.monitor("smart_value_"+attr, (serial,device), smart[device]["data"][attr]["value"]) pm.monitor("smart_thr_"+attr, (serial,device), smart[device]["data"][attr]["thr"]) pm.monitor("smart_worst_"+attr, (serial,device), smart[device]["data"][attr]["worst"]) except: pass else: try: pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr]) pm.monitor("smart_attributes", (serial,device,attr,"","","","",smart[device]["data"][attr]), smart[device]["data"][attr]) except: try: pm.add_monitor("smart_raw_"+attr, "", tags=("serial","instance")) pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr]) except: pass if "procMon" in sys.modules: try: pm.add_monitor("proc_summary", "", tags=("PID","CPU","VIRT","RAM","% RAM","THR","STARTTIME","RUNTIME","PARENT","STATE","COMM")) pm.add_monitor("proc_cpu", "%") pm.add_monitor("proc_memory_used", "B") pm.add_monitor("proc_memory_virt", "B") pm.add_monitor("proc_memory_percent", "%") pm.add_monitor("proc_page_fault_minor", "") pm.add_monitor("proc_page_fault_major", "") except: pass pm.delete_monitor("proc_summary") pm.delete_monitor("proc_cpu") pm.delete_monitor("proc_memory_used") pm.delete_monitor("proc_memory_virt") pm.delete_monitor("proc_memory_percent") pm.delete_monitor("proc_page_fault_minor") pm.delete_monitor("proc_page_fault_major") for pid in proc: try: pm.monitor("proc_summary", (proc[pid]["id"],proc[pid]["cpu"],str(int(proc[pid]["virt"]) / 1000)+" kB",str(int(proc[pid]["memory"]) / 1000)+" kB",round(float(proc[pid]["memory_percent"]),2),proc[pid]["threadcnt"],proc[pid]["starttime"],proc[pid]["runtime_seconds"],proc[pid]["parent_pid"],proc[pid]["state"],proc[pid]["comm"]), 1) pm.monitor("proc_cpu", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["cpu"]) pm.monitor("proc_memory_used", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["memory"]) pm.monitor("proc_memory_virt", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["virt"]) pm.monitor("proc_memory_percent", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["memory_percent"]) pm.monitor("proc_page_fault_minor", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["page_fault_minor"]) pm.monitor("proc_page_fault_major", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["page_fault_major"]) except: pass if "cpuinfo" in sys.modules: pm.monitor("cpu_info", (cpuinfo["vendor"],cpuinfo["model"],cpuinfo["cpus"]), 1) if "gpu" in sys.modules: pm.delete_monitor("gpu_info") pm.delete_monitor("gpu_util") pm.delete_monitor("gpu_throttle") for device in gpuinfo["gpu"]: pm.monitor("gpu_info", (gpuinfo["gpu"][device]["product_name"], gpuinfo["gpu"][device]["vbios_version"], gpuinfo["about"]["driver_version"], gpuinfo["gpu"][device]["performance_state"], device), 1) for item in gpuinfo["gpu"][device]["util"]: pm.monitor("gpu_util", (gpuinfo["gpu"][device]["product_name"], device, item), gpuinfo["gpu"][device]["util"][item]) for item in gpuinfo["gpu"][device]["throttle"]: pm.monitor("gpu_throttle", (gpuinfo["gpu"][device]["product_name"], device, item), gpuinfo["gpu"][device]["throttle"][item]) pm.monitor("gpu_fan_speed", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["fan_speed"]) pm.monitor("gpu_memory_used", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["memory_used"]) pm.monitor("gpu_memory_total", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["memory_total"]) pm.monitor("gpu_temp", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["temp"]) pm.monitor("gpu_power", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["power"]) pm.monitor("gpu_power_limit", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["power_limit"]) pm.monitor("gpu_gpu_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["gpu_clock"]) pm.monitor("gpu_mem_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["mem_clock"]) pm.monitor("gpu_sm_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["sm_clock"]) pm.monitor("gpu_video_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["video_clock"]) pm.monitor("gpu_gpu_clock_max", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["gpu_max_clock"]) pm.monitor("gpu_mem_clock_max", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["mem_max_clock"]) if "packages" in sys.modules: pm.delete_monitor("update_pkg_updatable") pm.delete_monitor("update_pkg_count") pm.delete_monitor("installed_pkg_count") pm.delete_monitor("installed_pkgs") pm.delete_monitor("installed_pkg_size") pm.monitor("update_pkg_count", (up["package_mgr"],), len(up["updatable"])) pm.monitor("installed_pkg_count", (up["package_mgr"],), len(up["installed"])) pm.monitor("installed_pkg_size", (up["package_mgr"],), up["total_size"]) try: for package in up["installed"]: pm.monitor("installed_pkgs", (package, up["installed"][package]["version"], up["installed"][package]["repository"]), up["installed"][package]["size"]) except: pass if len(up["updatable"]) > 0: for package in up["updatable"]: pm.monitor("update_pkg_updatable", (package, up["updatable"][package]["version"], up["updatable"][package]["repository"]), 1) mon_time = self_monitoring("prom_export", loop_start if mon_time == 0 else mon_time) loop_time_spent = self_monitoring("hw-mon-loop", loop_start) if (loop_start - datetime.datetime.now()).total_seconds() < 1: sleep(1 - (loop_start - datetime.datetime.now()).total_seconds())