initial commit
This commit is contained in:
690
main.py
Normal file
690
main.py
Normal file
@@ -0,0 +1,690 @@
|
||||
from time import sleep
|
||||
import datetime
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
|
||||
import promMon
|
||||
import mon_pkg_update
|
||||
|
||||
#===INIT========================================================================
|
||||
filedir = os.path.dirname(os.path.abspath(sys.argv[0]))
|
||||
|
||||
vols_to_scan = {}
|
||||
vols_to_scan["zfs"] = []
|
||||
vols_to_scan["non_zfs"] = []
|
||||
nics_to_scan = []
|
||||
SMART_data_update_period_seconds = 60
|
||||
last_SMART_read_timestamp = 0
|
||||
packages_data_update_period_seconds = 3600
|
||||
last_packages_read_timestamp = 0
|
||||
docker_data_update_period_seconds = 3600
|
||||
last_docker_read_timestamp = 0
|
||||
update_scan_period_seconds = 300
|
||||
last_update_scan_timestamp = 0
|
||||
update_from_git_tags = True
|
||||
gpu_vendor = "nVidia"
|
||||
|
||||
# load config.json file
|
||||
try:
|
||||
with open(filedir+"/config.json", "r") as f:
|
||||
config = json.loads(f.read())
|
||||
for module in config["modules"]:
|
||||
if "ryzenPower" in module:
|
||||
import ryzenPower
|
||||
if "kvmSensors" in module:
|
||||
import kvmSensors as ks
|
||||
if "sysinfo" in module:
|
||||
import sysinfo as si
|
||||
if "docker" in module:
|
||||
import docker as do
|
||||
if "SMART" in module:
|
||||
import SMART as sm
|
||||
if "sensors" in module:
|
||||
import sensors as lmsn
|
||||
if "intelPower" in module:
|
||||
import intelPower
|
||||
if "procMon" in module:
|
||||
import procMon as pr
|
||||
if "RAS" in module:
|
||||
import RAS
|
||||
if "cpuinfo" in module:
|
||||
import cpuinfo as cinfo
|
||||
if "gpu" in module:
|
||||
import gpu
|
||||
if "packages" in module:
|
||||
import packages
|
||||
try:
|
||||
if len(config["volumes"]["zfs"]) != 0:
|
||||
for i in config["volumes"]["zfs"]:
|
||||
vols_to_scan["zfs"].append(i)
|
||||
if len(config["volumes"]["non_zfs"]) != 0:
|
||||
for i in config["volumes"]["non_zfs"]:
|
||||
vols_to_scan["non_zfs"].append(i)
|
||||
except:
|
||||
print("No volumes object detected")
|
||||
try:
|
||||
if len(config["network"]["nics"]) != 0:
|
||||
for nic in config["network"]["nics"]:
|
||||
nics_to_scan.append(nic)
|
||||
except:
|
||||
print("No network object detected")
|
||||
try:
|
||||
gpu_vendor = config["gpu"]
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
print("No config file found, exitting")
|
||||
exit(255)
|
||||
|
||||
|
||||
proc_last = 0
|
||||
|
||||
if "promMon" not in sys.modules:
|
||||
print("Prometheus exporter helper not inported, exitting")
|
||||
exit(1)
|
||||
|
||||
pm = promMon.prometheus(port=9339, name="hw-monitor")
|
||||
|
||||
pm.add_monitor("self_loop_time", "us")
|
||||
pm.add_monitor("version", "", tags=("version",))
|
||||
|
||||
if "docker" in sys.modules:
|
||||
pm.add_monitor("docker_version", "")
|
||||
pm.add_monitor("docker_overall_info", "", tags=("module","type"))
|
||||
pm.add_monitor("docker_overall_size", "B", tags=("module","type"))
|
||||
pm.add_monitor("docker_images", "", tags=("id","containers","created","repository","unique_size"))
|
||||
pm.add_monitor("docker_containers", "", tags=("id","created","image","volumes","mounts","name","networks","runtime","state","status"))
|
||||
pm.add_monitor("docker_volumes", "", tags=("name","driver","links","mountpoint"))
|
||||
pm.add_monitor("docker_build_cache", "", tags=("id","type","created","in_use","last_use","shared","use_count"))
|
||||
|
||||
if "ryzenPower" in sys.modules:
|
||||
pm.add_monitor("package_power", "W")
|
||||
pm.add_monitor("core_total", "W")
|
||||
pm.add_monitor("core", "W")
|
||||
rp = ryzenPower.RyzenPower()
|
||||
|
||||
if "intelPower" in sys.modules:
|
||||
ip = intelPower.IntelPower()
|
||||
|
||||
if "kvmSensors" in sys.modules:
|
||||
pm.add_monitor("fan_rpm", "RPM")
|
||||
pm.add_monitor("fan_ok", "")
|
||||
pm.add_monitor("temp_celsius", "C")
|
||||
pm.add_monitor("temp_ok", "")
|
||||
pm.add_monitor("voltage", "V")
|
||||
pm.add_monitor("voltage_ok", "")
|
||||
|
||||
if "cpuinfo" in sys.modules:
|
||||
pm.add_monitor("cpu_info", "", tags=("vendor","model","cpus"))
|
||||
|
||||
if "sysinfo" in sys.modules:
|
||||
pm.add_monitor("disk_read", "B/s")
|
||||
pm.add_monitor("disk_write", "B/s")
|
||||
pm.add_monitor("disk_io_read", "iops")
|
||||
pm.add_monitor("disk_io_write", "iops")
|
||||
pm.add_monitor("disk_io_read_time", "ms")
|
||||
pm.add_monitor("disk_io_write_time", "ms")
|
||||
pm.add_monitor("disk_io_read_merged", "")
|
||||
pm.add_monitor("disk_io_write_merged", "")
|
||||
pm.add_monitor("disk_busy", "ms")
|
||||
pm.add_monitor("cpu_count", "")
|
||||
pm.add_monitor("cpu_frequency", "Hz")
|
||||
pm.add_monitor("cpu_usage", "%")
|
||||
pm.add_monitor("uptime", "s")
|
||||
pm.add_monitor("system_info", "", tags=("hostname","kernel","board"))
|
||||
pm.add_monitor("ip_addrs", "", tags=("interface","ip",))
|
||||
pm.add_monitor("user_sessions", "")
|
||||
pm.add_monitor("users", "", tags=("user","from"))
|
||||
|
||||
if "SMART" in sys.modules:
|
||||
pm.add_monitor("smart_attributes", "", tags=("serial","device","attribute","id","value","thres","worst","raw"))
|
||||
|
||||
if "gpu" in sys.modules:
|
||||
pm.add_monitor("gpu_info", "", tags=("name","vbios","driver","pstate","uuid"))
|
||||
pm.add_monitor("gpu_util", "%", tags=("name","uuid","stat"))
|
||||
pm.add_monitor("gpu_throttle", "", tags=("name","uuid","stat"))
|
||||
pm.add_monitor("gpu_memory_used", "MB", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_memory_total", "MB", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_power", "W", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_power_limit", "W", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_temp", "C", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_fan_speed", "%", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_gpu_clock", "Hz", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_mem_clock", "Hz", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_sm_clock", "Hz", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_video_clock", "Hz", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_gpu_clock_max", "Hz", tags=("name","uuid"))
|
||||
pm.add_monitor("gpu_mem_clock_max", "Hz", tags=("name","uuid"))
|
||||
|
||||
if "packages" in sys.modules:
|
||||
pm.add_monitor("update_pkg_count", "", tags=("package_mgr",))
|
||||
pm.add_monitor("update_pkg_updatable", "", tags=("package","version","repository"))
|
||||
pm.add_monitor("installed_pkg_count", "", tags=("package_mgr",))
|
||||
pm.add_monitor("installed_pkgs", "", tags=("package","version","repository"))
|
||||
pm.add_monitor("installed_pkg_size", "", tags=("package_mgr",))
|
||||
|
||||
def self_monitoring(name, start):
|
||||
end = datetime.datetime.now()
|
||||
task_time = end - start
|
||||
pm.monitor("self_loop_time", (name,), task_time.microseconds + (task_time.seconds * 1000000))
|
||||
|
||||
return datetime.datetime.now()
|
||||
|
||||
while True:
|
||||
mon_time = 0
|
||||
loop_start = datetime.datetime.now()
|
||||
|
||||
#===VARIABLE INIT===============================================================
|
||||
|
||||
if "ryzenPower" in sys.modules:
|
||||
package, cores = rp.measure_nonblocking()
|
||||
mon_time = self_monitoring("zenpower", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "kvmSensors" in sys.modules:
|
||||
kvm = ks.readSensors()
|
||||
mon_time = self_monitoring("kvm", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "sysinfo" in sys.modules:
|
||||
disks = si.getDisk()
|
||||
mon_time = self_monitoring("disks", loop_start if mon_time == 0 else mon_time)
|
||||
cpu = si.getCPU()
|
||||
mon_time = self_monitoring("cpu", loop_start if mon_time == 0 else mon_time)
|
||||
memory = si.getMemory()
|
||||
mon_time = self_monitoring("memory", loop_start if mon_time == 0 else mon_time)
|
||||
try:
|
||||
partitions = si.getPartitions()
|
||||
mon_time = self_monitoring("partitions", loop_start if mon_time == 0 else mon_time)
|
||||
except:
|
||||
partitions = 0
|
||||
try:
|
||||
zfs = si.getZFS()
|
||||
mon_time = self_monitoring("zfs", loop_start if mon_time == 0 else mon_time)
|
||||
except:
|
||||
zfs = 0
|
||||
uptime = si.getUptime()
|
||||
mon_time = self_monitoring("uptime", loop_start if mon_time == 0 else mon_time)
|
||||
network = si.getNetwork()
|
||||
ipaddrs = si.getIP()
|
||||
mon_time = self_monitoring("network", loop_start if mon_time == 0 else mon_time)
|
||||
systeminfo = si.getSysInfo()
|
||||
mon_time = self_monitoring("systeminfo", loop_start if mon_time == 0 else mon_time)
|
||||
users = si.users()
|
||||
mon_time = self_monitoring("users", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "SMART" in sys.modules:
|
||||
if last_SMART_read_timestamp == 0:
|
||||
smart = sm.getAllDeviceAttributes()
|
||||
last_SMART_read_timestamp = datetime.datetime.now()
|
||||
else:
|
||||
timedelta = (datetime.datetime.now() - last_SMART_read_timestamp).total_seconds()
|
||||
if timedelta >= SMART_data_update_period_seconds:
|
||||
smart = sm.getAllDeviceAttributes()
|
||||
last_SMART_read_timestamp = datetime.datetime.now()
|
||||
mon_time = self_monitoring("smart", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "docker" in sys.modules:
|
||||
containers = do.health()
|
||||
try:
|
||||
if last_docker_read_timestamp == 0:
|
||||
docker_info = do.getInfo()
|
||||
docker_size = do.getSize()
|
||||
last_docker_read_timestamp = datetime.datetime.now()
|
||||
else:
|
||||
timedelta = (datetime.datetime.now() - last_docker_read_timestamp).total_seconds()
|
||||
if timedelta >= docker_data_update_period_seconds:
|
||||
docker_info = do.getInfo()
|
||||
docker_size = do.getSize()
|
||||
last_docker_read_timestamp = datetime.datetime.now()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
mon_time = self_monitoring("containers", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "sensors" in sys.modules:
|
||||
temperatures = lmsn.getTemps()
|
||||
mon_time = self_monitoring("temperatures", loop_start if mon_time == 0 else mon_time)
|
||||
fans = lmsn.getFans()
|
||||
mon_time = self_monitoring("fans", loop_start if mon_time == 0 else mon_time)
|
||||
voltages = lmsn.getVoltages()
|
||||
mon_time = self_monitoring("voltage", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "intelPower" in sys.modules:
|
||||
package, cores = ip.measure_nonblocking()
|
||||
mon_time = self_monitoring("cpu_power", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "RAS" in sys.modules:
|
||||
ras = RAS.readRAS()
|
||||
mon_time = self_monitoring("ras", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "procMon" in sys.modules:
|
||||
proc = pr.exportProcesses(proc_last)
|
||||
proc_last = proc
|
||||
mon_time = self_monitoring("proc", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "cpuinfo" in sys.modules:
|
||||
cpuinfo = cinfo.getCpuInfo()
|
||||
mon_time = self_monitoring("cpuinfo", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "gpu" in sys.modules:
|
||||
gpuinfo = gpu.readGpu(gpu_vendor)
|
||||
mon_time = self_monitoring("gpu", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if "packages" in sys.modules:
|
||||
if last_packages_read_timestamp == 0:
|
||||
up = packages.getPackages()
|
||||
last_packages_read_timestamp = datetime.datetime.now()
|
||||
else:
|
||||
timedelta = (datetime.datetime.now() - last_packages_read_timestamp).total_seconds()
|
||||
if timedelta >= packages_data_update_period_seconds:
|
||||
up = packages.getPackages()
|
||||
last_packages_read_timestamp = datetime.datetime.now()
|
||||
mon_time = self_monitoring("packages", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
if last_update_scan_timestamp == 0:
|
||||
version = mon_pkg_update.getCurrentTag()
|
||||
if update_from_git_tags:
|
||||
mon_pkg_update.update()
|
||||
last_update_scan_timestamp = datetime.datetime.now()
|
||||
else:
|
||||
timedelta = (datetime.datetime.now() - last_update_scan_timestamp).total_seconds()
|
||||
if timedelta >= update_scan_period_seconds:
|
||||
version = mon_pkg_update.getCurrentTag()
|
||||
if update_from_git_tags:
|
||||
mon_pkg_update.update()
|
||||
last_update_scan_timestamp = datetime.datetime.now()
|
||||
mon_time = self_monitoring("version_check", loop_start if mon_time == 0 else mon_time)
|
||||
|
||||
#===MOVE VARS TO PROMETHEUS EXPORTER============================================
|
||||
|
||||
pm.monitor("version", (version,), 1)
|
||||
|
||||
if "sysinfo" in sys.modules:
|
||||
pm.monitor("uptime", ("sensors",), uptime)
|
||||
pm.monitor("cpu_count", ("sensors",), cpu["cpu_count"])
|
||||
pm.monitor("cpu_usage", ("sensors",), cpu["usage"])
|
||||
for disk in disks:
|
||||
pm.monitor("disk_read", (disk,), disks[disk]["read"])
|
||||
pm.monitor("disk_write", (disk,), disks[disk]["write"])
|
||||
pm.monitor("disk_io_read", (disk,), disks[disk]["io_read"])
|
||||
pm.monitor("disk_io_write", (disk,), disks[disk]["io_write"])
|
||||
pm.monitor("disk_io_read_time", (disk,), disks[disk]["io_read_time"])
|
||||
pm.monitor("disk_io_write_time", (disk,), disks[disk]["io_write_time"])
|
||||
pm.monitor("disk_io_read_merged", (disk,), disks[disk]["io_read_merged"])
|
||||
pm.monitor("disk_io_write_merged", (disk,), disks[disk]["io_write_merged"])
|
||||
pm.monitor("disk_busy", (disk,), disks[disk]["busy"])
|
||||
|
||||
for core in cpu["frequency"]:
|
||||
pm.monitor("cpu_frequency", (str(core),), cpu["frequency"][core])
|
||||
|
||||
for t in cpu["time_percent"]:
|
||||
try:
|
||||
pm.monitor("cpu_time_percent", (t,), cpu["time_percent"][t])
|
||||
except:
|
||||
pm.add_monitor("cpu_time_percent", "%")
|
||||
pm.monitor("cpu_time_percent", (t,), cpu["time_percent"][t])
|
||||
|
||||
for i in memory:
|
||||
try:
|
||||
pm.monitor("memory_"+i, ("memory",), memory[i])
|
||||
except:
|
||||
pm.add_monitor("memory_"+i, "%")
|
||||
pm.monitor("memory_"+i, ("memory",), memory[i])
|
||||
|
||||
if zfs != 0:
|
||||
for pool in zfs:
|
||||
if len(vols_to_scan["zfs"]) != 0:
|
||||
if pool not in vols_to_scan["zfs"]:
|
||||
continue
|
||||
try:
|
||||
pm.monitor("zfs_state", (pool,), zfs[pool]["state"])
|
||||
pm.monitor("zfs_size", (pool,), zfs[pool]["size"])
|
||||
pm.monitor("zfs_used", (pool,), zfs[pool]["used"])
|
||||
pm.monitor("zfs_free", (pool,), zfs[pool]["free"])
|
||||
pm.monitor("zfs_fragmentation", (pool,), zfs[pool]["fragmentation"])
|
||||
pm.monitor("zfs_dedup", (pool,), zfs[pool]["dedup"])
|
||||
except:
|
||||
pm.add_monitor("zfs_state", "")
|
||||
pm.add_monitor("zfs_size", "B")
|
||||
pm.add_monitor("zfs_used", "B")
|
||||
pm.add_monitor("zfs_free", "B")
|
||||
pm.add_monitor("zfs_fragmentation", "%")
|
||||
pm.add_monitor("zfs_dedup", "")
|
||||
pm.monitor("zfs_state", (pool,), zfs[pool]["state"])
|
||||
pm.monitor("zfs_size", (pool,), zfs[pool]["size"])
|
||||
pm.monitor("zfs_used", (pool,), zfs[pool]["used"])
|
||||
pm.monitor("zfs_free", (pool,), zfs[pool]["free"])
|
||||
pm.monitor("zfs_fragmentation", (pool,), zfs[pool]["fragmentation"])
|
||||
pm.monitor("zfs_dedup", (pool,), zfs[pool]["dedup"])
|
||||
|
||||
for nic in network:
|
||||
if len(nics_to_scan) != 0:
|
||||
if nic not in nics_to_scan:
|
||||
continue
|
||||
try:
|
||||
pm.monitor("network_rx", (nic,), network[nic]["rx"])
|
||||
pm.monitor("network_tx", (nic,), network[nic]["tx"])
|
||||
pm.monitor("network_err_rx", (nic,), network[nic]["err_rx"])
|
||||
pm.monitor("network_err_tx", (nic,), network[nic]["err_tx"])
|
||||
pm.monitor("network_drop_rx", (nic,), network[nic]["drop_rx"])
|
||||
pm.monitor("network_drop_tx", (nic,), network[nic]["drop_tx"])
|
||||
pm.monitor("network_packet_rx", (nic,), network[nic]["packet_rx"])
|
||||
pm.monitor("network_packet_tx", (nic,), network[nic]["packet_tx"])
|
||||
except:
|
||||
pm.add_monitor("network_rx", "B")
|
||||
pm.add_monitor("network_tx", "B")
|
||||
pm.add_monitor("network_err_rx", "")
|
||||
pm.add_monitor("network_err_tx", "")
|
||||
pm.add_monitor("network_drop_rx", "")
|
||||
pm.add_monitor("network_drop_tx", "")
|
||||
pm.add_monitor("network_packet_rx", "")
|
||||
pm.add_monitor("network_packet_tx", "")
|
||||
pm.monitor("network_rx", (nic,), network[nic]["rx"])
|
||||
pm.monitor("network_tx", (nic,), network[nic]["tx"])
|
||||
pm.monitor("network_err_rx", (nic,), network[nic]["err_rx"])
|
||||
pm.monitor("network_err_tx", (nic,), network[nic]["err_tx"])
|
||||
pm.monitor("network_drop_rx", (nic,), network[nic]["drop_rx"])
|
||||
pm.monitor("network_drop_tx", (nic,), network[nic]["drop_tx"])
|
||||
pm.monitor("network_packet_rx", (nic,), network[nic]["packet_rx"])
|
||||
pm.monitor("network_packet_tx", (nic,), network[nic]["packet_tx"])
|
||||
|
||||
if partitions != 0:
|
||||
for part in partitions:
|
||||
if len(vols_to_scan["non_zfs"]) != 0:
|
||||
if part not in vols_to_scan["non_zfs"]:
|
||||
continue
|
||||
try:
|
||||
pm.monitor("partition_size", (part,), partitions[part]["size"])
|
||||
pm.monitor("partition_used", (part,), partitions[part]["used"])
|
||||
pm.monitor("partition_free", (part,), partitions[part]["free"])
|
||||
except:
|
||||
pm.add_monitor("partition_size", "B")
|
||||
pm.add_monitor("partition_used", "B")
|
||||
pm.add_monitor("partition_free", "B")
|
||||
pm.monitor("partition_size", (part,), partitions[part]["size"])
|
||||
pm.monitor("partition_used", (part,), partitions[part]["used"])
|
||||
pm.monitor("partition_free", (part,), partitions[part]["free"])
|
||||
|
||||
pm.monitor("system_info", (systeminfo["hostname"],systeminfo["kernel"],systeminfo["board"]), 1)
|
||||
|
||||
pm.delete_monitor("user_sessions")
|
||||
pm.delete_monitor("users")
|
||||
|
||||
try:
|
||||
for user in users:
|
||||
pm.monitor("user_sessions", (user,), users[user]["sessions"])
|
||||
for session in users[user]["session"]:
|
||||
pm.monitor("users", (user,users[user]["session"][session]["from"]), 1)
|
||||
except:
|
||||
pass
|
||||
|
||||
pm.delete_monitor("ip_addrs")
|
||||
for interface in ipaddrs:
|
||||
pm.monitor("ip_addrs", (interface,ipaddrs[interface]), 1)
|
||||
|
||||
if "ryzenPower" in sys.modules:
|
||||
pm.monitor("package_power", ("sensors",), package)
|
||||
core_total = 0
|
||||
for core in cores:
|
||||
pm.monitor("core", (str(int(core/2)),), cores[core])
|
||||
core_total += cores[core]
|
||||
pm.monitor("core_total", ("sensors",), core_total)
|
||||
|
||||
if "kvmSensors" in sys.modules:
|
||||
for fan in kvm["fans"]:
|
||||
pm.monitor("fan_rpm", (kvm["fans"][fan]["id"],), kvm["fans"][fan]["rpm"])
|
||||
if kvm["fans"][fan]["status"] == "ok":
|
||||
pm.monitor("fan_ok", (kvm["fans"][fan]["id"],), 1)
|
||||
else:
|
||||
pm.monitor("fan_ok", (kvm["fans"][fan]["id"],), 0)
|
||||
|
||||
for temp in kvm["temp"]:
|
||||
pm.monitor("temp_celsius", (kvm["temp"][temp]["id"],), kvm["temp"][temp]["temp"])
|
||||
if kvm["temp"][temp]["status"] == "ok":
|
||||
pm.monitor("temp_ok", (kvm["temp"][temp]["id"],), 1)
|
||||
else:
|
||||
pm.monitor("temp_ok", (kvm["temp"][temp]["id"],), 0)
|
||||
|
||||
for volt in kvm["volt"]:
|
||||
pm.monitor("voltage", (kvm["volt"][volt]["id"],), kvm["volt"][volt]["voltage"])
|
||||
if kvm["volt"][volt]["status"] == "ok":
|
||||
pm.monitor("voltage_ok", (kvm["volt"][volt]["id"],), 1)
|
||||
else:
|
||||
pm.monitor("voltage_ok", (kvm["volt"][volt]["id"],), 0)
|
||||
|
||||
if "docker" in sys.modules:
|
||||
pm.delete_monitor("docker_version")
|
||||
pm.delete_monitor("docker_images")
|
||||
pm.delete_monitor("docker_containers")
|
||||
pm.delete_monitor("docker_volumes")
|
||||
pm.delete_monitor("docker_build_cache")
|
||||
|
||||
for container in containers:
|
||||
try:
|
||||
pm.monitor("docker_status", (container,), containers[container]["status"])
|
||||
pm.monitor("docker_health", (container,), containers[container]["health"])
|
||||
except:
|
||||
pm.add_monitor("docker_status", "")
|
||||
pm.add_monitor("docker_health", "")
|
||||
pm.monitor("docker_status", (container,), containers[container]["status"])
|
||||
pm.monitor("docker_health", (container,), containers[container]["health"])
|
||||
|
||||
try:
|
||||
pm.monitor("docker_version", (docker_info["version"],), 1)
|
||||
for module in docker_size:
|
||||
pm.monitor("docker_overall_info", (module,"count_total"), int(docker_size[module]["count_total"]))
|
||||
pm.monitor("docker_overall_info", (module,"count_active"), int(docker_size[module]["count_active"]))
|
||||
pm.monitor("docker_overall_size", (module,"used"), docker_size[module]["used"])
|
||||
pm.monitor("docker_overall_size", (module,"reclaimable"), docker_size[module]["reclaimable"])
|
||||
|
||||
for image in docker_info["images"]:
|
||||
pm.monitor("docker_images", (image,docker_info["images"][image]["containers"],docker_info["images"][image]["created"],docker_info["images"][image]["repository"],docker_info["images"][image]["unique_size"]), docker_info["images"][image]["size"])
|
||||
|
||||
for container in docker_info["containers"]:
|
||||
pm.monitor("docker_containers", (container,docker_info["containers"][container]["created"],docker_info["containers"][container]["image"],docker_info["containers"][container]["volumes"],docker_info["containers"][container]["mounts"],docker_info["containers"][container]["name"],docker_info["containers"][container]["networks"],docker_info["containers"][container]["runtime"],docker_info["containers"][container]["state"],docker_info["containers"][container]["status"]), docker_info["containers"][container]["size"])
|
||||
|
||||
for volume in docker_info["volumes"]:
|
||||
pm.monitor("docker_volumes", (volume,docker_info["volumes"][volume]["driver"],docker_info["volumes"][volume]["links"],docker_info["volumes"][volume]["mountpoint"]), docker_info["volumes"][volume]["size"])
|
||||
|
||||
for cache in docker_info["buildcache"]:
|
||||
pm.monitor("docker_build_cache", (cache,docker_info["buildcache"][cache]["type"],docker_info["buildcache"][cache]["created"],docker_info["buildcache"][cache]["in_use"],docker_info["buildcache"][cache]["last_use"],docker_info["buildcache"][cache]["shared"],docker_info["buildcache"][cache]["use_count"]), docker_info["buildcache"][cache]["size"])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
if "sensors" in sys.modules:
|
||||
for temp in temperatures:
|
||||
try:
|
||||
for sensor in temperatures[temp]:
|
||||
if "coretemp" in temp:
|
||||
pm.monitor("temp_celsius_lm", (str(sensor),), temperatures[temp][sensor])
|
||||
else:
|
||||
pm.monitor("temp_celsius_lm", (str(temp)+'_'+str(sensor),), temperatures[temp][sensor])
|
||||
except:
|
||||
pm.add_monitor("temp_celsius_lm", "C")
|
||||
for sensor in temperatures[temp]:
|
||||
if "coretemp" in temp:
|
||||
pm.monitor("temp_celsius_lm", (str(sensor),), temperatures[temp][sensor])
|
||||
else:
|
||||
pm.monitor("temp_celsius_lm", (str(temp)+'_'+str(sensor),), temperatures[temp][sensor])
|
||||
|
||||
for fan in fans:
|
||||
try:
|
||||
for sensor in fans[fan]:
|
||||
pm.monitor("fans_lm", (str(sensor),), fans[fan][sensor])
|
||||
except:
|
||||
pm.add_monitor("fans_lm", "RPM")
|
||||
for sensor in fans[fan]:
|
||||
pm.monitor("fans_lm", (str(sensor),), fans[fan][sensor])
|
||||
|
||||
for voltage in voltages:
|
||||
try:
|
||||
for sensor in voltages[voltage]:
|
||||
pm.monitor("voltages_lm", (str(sensor),), voltages[voltage][sensor])
|
||||
except:
|
||||
pm.add_monitor("voltages_lm", "V")
|
||||
for sensor in voltages[voltage]:
|
||||
pm.monitor("voltages_lm", (str(sensor),), voltages[voltage][sensor])
|
||||
|
||||
if "intelPower" in sys.modules:
|
||||
try:
|
||||
pm.monitor("intel_cpu_power", ("core",), cores)
|
||||
pm.monitor("intel_cpu_power", ("package",), package)
|
||||
except:
|
||||
pm.add_monitor("intel_cpu_power", "W")
|
||||
pm.monitor("intel_cpu_power", ("core",), cores)
|
||||
pm.monitor("intel_cpu_power", ("package",), package)
|
||||
|
||||
if "RAS" in sys.modules:
|
||||
for item in ras:
|
||||
try:
|
||||
for tag in ras[item]:
|
||||
if tag == "total_errors":
|
||||
pm.monitor("ras_total", (item.replace(' ','_'),), ras[item][tag])
|
||||
else:
|
||||
for error in ras[item][tag]:
|
||||
pm.monitor("ras_"+error, (item.replace(' ','_')+"_"+tag,), ras[item][tag][error])
|
||||
except:
|
||||
for tag in ras[item]:
|
||||
if tag == "total_errors":
|
||||
try:
|
||||
pm.add_monitor("ras_total", "")
|
||||
pm.monitor("ras_total", (item.replace(' ','_'),), ras[item][tag])
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
for error in ras[item][tag]:
|
||||
try:
|
||||
pm.add_monitor("ras_"+error, "")
|
||||
pm.monitor("ras_"+error, (item.replace(' ','_')+"_"+tag,), ras[item][tag][error])
|
||||
except:
|
||||
pass
|
||||
|
||||
if "SMART" in sys.modules:
|
||||
try:
|
||||
pm.add_monitor("smart_bytes_written", "B", tags=("serial","instance"))
|
||||
except:
|
||||
pass
|
||||
|
||||
pm.delete_monitor("smart_attributes")
|
||||
|
||||
for device in smart:
|
||||
serial = smart[device]["serial_number"]
|
||||
|
||||
pm.monitor("smart_bytes_written", (serial,device), smart[device]["bytes_written"])
|
||||
|
||||
for attr in smart[device]["data"]:
|
||||
if smart[device]["type"] == "ATA":
|
||||
try:
|
||||
pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr]["raw"])
|
||||
pm.monitor("smart_value_"+attr, (serial,device), smart[device]["data"][attr]["value"])
|
||||
pm.monitor("smart_thr_"+attr, (serial,device), smart[device]["data"][attr]["thr"])
|
||||
pm.monitor("smart_worst_"+attr, (serial,device), smart[device]["data"][attr]["worst"])
|
||||
pm.monitor("smart_attributes", (serial,device,attr,smart[device]["data"][attr]["id"],smart[device]["data"][attr]["value"],smart[device]["data"][attr]["thr"],smart[device]["data"][attr]["worst"],smart[device]["data"][attr]["raw"]), smart[device]["data"][attr]["raw"])
|
||||
except:
|
||||
try:
|
||||
pm.add_monitor("smart_raw_"+attr, "", tags=("serial","instance"))
|
||||
pm.add_monitor("smart_value_"+attr, "", tags=("serial","instance"))
|
||||
pm.add_monitor("smart_thr_"+attr, "", tags=("serial","instance"))
|
||||
pm.add_monitor("smart_worst_"+attr, "", tags=("serial","instance"))
|
||||
pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr]["raw"])
|
||||
pm.monitor("smart_value_"+attr, (serial,device), smart[device]["data"][attr]["value"])
|
||||
pm.monitor("smart_thr_"+attr, (serial,device), smart[device]["data"][attr]["thr"])
|
||||
pm.monitor("smart_worst_"+attr, (serial,device), smart[device]["data"][attr]["worst"])
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr])
|
||||
pm.monitor("smart_attributes", (serial,device,attr,"","","","",smart[device]["data"][attr]), smart[device]["data"][attr])
|
||||
except:
|
||||
try:
|
||||
pm.add_monitor("smart_raw_"+attr, "", tags=("serial","instance"))
|
||||
pm.monitor("smart_raw_"+attr, (serial,device), smart[device]["data"][attr])
|
||||
except:
|
||||
pass
|
||||
|
||||
if "procMon" in sys.modules:
|
||||
try:
|
||||
pm.add_monitor("proc_summary", "", tags=("PID","CPU","VIRT","RAM","% RAM","THR","STARTTIME","RUNTIME","PARENT","STATE","COMM"))
|
||||
pm.add_monitor("proc_cpu", "%")
|
||||
pm.add_monitor("proc_memory_used", "B")
|
||||
pm.add_monitor("proc_memory_virt", "B")
|
||||
pm.add_monitor("proc_memory_percent", "%")
|
||||
pm.add_monitor("proc_page_fault_minor", "")
|
||||
pm.add_monitor("proc_page_fault_major", "")
|
||||
except:
|
||||
pass
|
||||
|
||||
pm.delete_monitor("proc_summary")
|
||||
pm.delete_monitor("proc_cpu")
|
||||
pm.delete_monitor("proc_memory_used")
|
||||
pm.delete_monitor("proc_memory_virt")
|
||||
pm.delete_monitor("proc_memory_percent")
|
||||
pm.delete_monitor("proc_page_fault_minor")
|
||||
pm.delete_monitor("proc_page_fault_major")
|
||||
|
||||
for pid in proc:
|
||||
try:
|
||||
pm.monitor("proc_summary", (proc[pid]["id"],proc[pid]["cpu"],str(int(proc[pid]["virt"]) / 1000)+" kB",str(int(proc[pid]["memory"]) / 1000)+" kB",round(float(proc[pid]["memory_percent"]),2),proc[pid]["threadcnt"],proc[pid]["starttime"],proc[pid]["runtime_seconds"],proc[pid]["parent_pid"],proc[pid]["state"],proc[pid]["comm"]), 1)
|
||||
pm.monitor("proc_cpu", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["cpu"])
|
||||
pm.monitor("proc_memory_used", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["memory"])
|
||||
pm.monitor("proc_memory_virt", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["virt"])
|
||||
pm.monitor("proc_memory_percent", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["memory_percent"])
|
||||
pm.monitor("proc_page_fault_minor", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["page_fault_minor"])
|
||||
pm.monitor("proc_page_fault_major", (str(proc[pid]["id"])+"_"+proc[pid]["comm"],), proc[pid]["page_fault_major"])
|
||||
except:
|
||||
pass
|
||||
|
||||
if "cpuinfo" in sys.modules:
|
||||
pm.monitor("cpu_info", (cpuinfo["vendor"],cpuinfo["model"],cpuinfo["cpus"]), 1)
|
||||
|
||||
if "gpu" in sys.modules:
|
||||
pm.delete_monitor("gpu_info")
|
||||
pm.delete_monitor("gpu_util")
|
||||
pm.delete_monitor("gpu_throttle")
|
||||
|
||||
for device in gpuinfo["gpu"]:
|
||||
pm.monitor("gpu_info", (gpuinfo["gpu"][device]["product_name"], gpuinfo["gpu"][device]["vbios_version"], gpuinfo["about"]["driver_version"], gpuinfo["gpu"][device]["performance_state"], device), 1)
|
||||
|
||||
for item in gpuinfo["gpu"][device]["util"]:
|
||||
pm.monitor("gpu_util", (gpuinfo["gpu"][device]["product_name"], device, item), gpuinfo["gpu"][device]["util"][item])
|
||||
for item in gpuinfo["gpu"][device]["throttle"]:
|
||||
pm.monitor("gpu_throttle", (gpuinfo["gpu"][device]["product_name"], device, item), gpuinfo["gpu"][device]["throttle"][item])
|
||||
|
||||
pm.monitor("gpu_fan_speed", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["fan_speed"])
|
||||
pm.monitor("gpu_memory_used", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["memory_used"])
|
||||
pm.monitor("gpu_memory_total", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["memory_total"])
|
||||
pm.monitor("gpu_temp", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["temp"])
|
||||
pm.monitor("gpu_power", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["power"])
|
||||
pm.monitor("gpu_power_limit", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["power_limit"])
|
||||
pm.monitor("gpu_gpu_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["gpu_clock"])
|
||||
pm.monitor("gpu_mem_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["mem_clock"])
|
||||
pm.monitor("gpu_sm_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["sm_clock"])
|
||||
pm.monitor("gpu_video_clock", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["video_clock"])
|
||||
pm.monitor("gpu_gpu_clock_max", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["gpu_max_clock"])
|
||||
pm.monitor("gpu_mem_clock_max", (gpuinfo["gpu"][device]["product_name"], device), gpuinfo["gpu"][device]["mem_max_clock"])
|
||||
|
||||
if "packages" in sys.modules:
|
||||
pm.delete_monitor("update_pkg_updatable")
|
||||
pm.delete_monitor("update_pkg_count")
|
||||
pm.delete_monitor("installed_pkg_count")
|
||||
pm.delete_monitor("installed_pkgs")
|
||||
pm.delete_monitor("installed_pkg_size")
|
||||
|
||||
pm.monitor("update_pkg_count", (up["package_mgr"],), len(up["updatable"]))
|
||||
pm.monitor("installed_pkg_count", (up["package_mgr"],), len(up["installed"]))
|
||||
pm.monitor("installed_pkg_size", (up["package_mgr"],), up["total_size"])
|
||||
|
||||
try:
|
||||
for package in up["installed"]:
|
||||
pm.monitor("installed_pkgs", (package, up["installed"][package]["version"], up["installed"][package]["repository"]), up["installed"][package]["size"])
|
||||
except:
|
||||
pass
|
||||
if len(up["updatable"]) > 0:
|
||||
for package in up["updatable"]:
|
||||
pm.monitor("update_pkg_updatable", (package, up["updatable"][package]["version"], up["updatable"][package]["repository"]), 1)
|
||||
|
||||
mon_time = self_monitoring("prom_export", loop_start if mon_time == 0 else mon_time)
|
||||
loop_time_spent = self_monitoring("hw-mon-loop", loop_start)
|
||||
|
||||
if (loop_start - datetime.datetime.now()).total_seconds() < 1:
|
||||
sleep(1 - (loop_start - datetime.datetime.now()).total_seconds())
|
||||
Reference in New Issue
Block a user