Your IP : 216.73.217.110
# Copyright (c) Cloud Linux Software, Inc
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENCE.TXT
"""Doctor v2 vitals collector (KPT-5984).
Python port of the kcdoctor.sh (1.0-8) vitals collection: gathers the
same data set into a DataPackage-based archive processed by the patch
server generic uploads pipeline.
Scalar and short values are consolidated into a single ``doctor.json``
(the eportal doctor precedent), one field each, instead of a tiny
archive entry per value; bulk data (logs, command dumps, package lists,
sysfs tunables, crash artifacts) stays as separate archive entries. The
1:1 inventory of collected items lives in
docs/features/KPT-5984-doctor-v2/parity-inventory.md.
"""
import fnmatch
import glob
import os
import re
import socket
from kcarectl import auth, config, delivery_kit, http_utils, kcare, log_utils, platform_utils, serverid, utils
from kcarectl.process_utils import run_command
from kcarectl.py23 import HTTPError, URLError, httplib
if False: # pragma: no cover
from typing import Any, Callable, Dict, List, Optional # noqa: F401
DOCTOR_VERSION = '2.0-1'
GRUB2_CFG = '/boot/grub2/grub.cfg'
KDUMP_CONF = '/etc/kdump.conf'
DPKG_LOG = '/var/log/dpkg.log'
APT_SOURCES_GLOB = '/etc/apt/sources.list*'
YUM_REPOS_GLOB = '/etc/yum.repos.d/*'
LIBCARE_LOGS_GLOB = '/var/log/libcare/*.log'
LOG_TAIL_LINES = 10000
SYSFS_ROOT = '/sys'
SYSFS_NAME_PATTERNS = ('enable*', 'nr_*', 'max_*', '*cnt*')
# plain prefix match, mirroring the kcdoctor.sh get_sysfs_info_detail()
# `! -path "<prefix>*"` find filters
SYSFS_EXCLUDE_PREFIXES = (
'/sys/kernel/debug/tracing/events',
'/sys/kernel/tracing/events',
'/sys/devices',
)
CRASH_DUMP_PATTERNS = ('[0-9]*.log', 'kmsg*log')
class KcarectlDoctorPackage(delivery_kit.DataPackage):
data_type = 'kcarectl-doctor'
upload_uri = '/upload/kcarectl-doctor/'
@property
def max_size(self):
# type: () -> int
return config.DOCTOR_REPORT_MAX_SIZE_BYTES
# --- bulk items: one archive file each ---
class CollectionSpec(object):
"""A single bulk vitals item ported from kcdoctor.sh.
Collection failures are recorded in errors.log and never abort the
report: an unhandled exception inside the package context manager
would remove the whole archive.
"""
def __init__(self, arcname, collect):
# type: (str, Callable[[delivery_kit.DataPackage, str], None]) -> None
self.arcname = arcname
self._collect = collect
def apply(self, data_package):
# type: (delivery_kit.DataPackage) -> None
try:
self._collect(data_package, self.arcname)
except Exception as e:
data_package.log_error('failed to collect {0}: {1}'.format(self.arcname, e))
def run_spec(arcname, cmd):
# type: (str, str) -> CollectionSpec
"""`run "<cmd>"` kcdoctor.sh helper (no shell features)"""
def collect(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
data_package.add_stdout(arcname, cmd)
return CollectionSpec(arcname, collect)
def dump_spec(arcname, path):
# type: (str, str) -> CollectionSpec
"""`dump "<path>"` kcdoctor.sh helper for regular files"""
def collect(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
data_package.add_file(arcname, src_path=path)
return CollectionSpec(arcname, collect)
def proc_dump_spec(arcname, path):
# type: (str, str) -> CollectionSpec
"""`dump "<path>"` for /proc and /sys files.
tar reports them as 0-size so they are first read into memory
(see the same approach in anomaly.prepare_kernel_anomaly_report).
"""
def collect(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
if not os.path.exists(path):
data_package.log_error('file not found: {0}'.format(path))
return
with open(path, 'rb') as f:
data_package.add_file(arcname, data_bytes=f.read())
return CollectionSpec(arcname, collect)
def callable_spec(arcname, fn):
# type: (str, Callable[[delivery_kit.DataPackage, str], None]) -> CollectionSpec
return CollectionSpec(arcname, fn)
# --- scalar/short items: one doctor.json field each ---
class FieldSpec(object):
"""A single doctor.json field ported from kcdoctor.sh.
Collection failures are recorded in errors.log and never abort the
report; a collector returning None omits the field (e.g. a
distro-specific source file absent on this system).
"""
def __init__(self, key, collect):
# type: (str, Callable[[], Any]) -> None
self.key = key
self._collect = collect
def apply(self, data_package, report):
# type: (delivery_kit.DataPackage, Dict[str, Any]) -> None
try:
value = self._collect()
except Exception as e:
data_package.log_error('failed to collect {0}: {1}'.format(self.key, e))
return
if value is not None:
report[self.key] = value
def read_field(path):
# type: (str) -> Callable[[], Optional[str]]
"""`dump "<path>"` as a doctor.json string field.
Read in binary (non-utf8 safe) and decoded leniently; an absent
distro-specific source file simply omits the field (None).
"""
def collect():
# type: () -> Optional[str]
if not os.path.exists(path):
return None
with open(path, 'rb') as f:
return f.read().decode('utf-8', 'replace').strip()
return collect
def grep_field(path, pattern):
# type: (str, str) -> Callable[[], Optional[List[str]]]
"""`grep <pattern> <path>` as a doctor.json list-of-lines field.
Match on bytes (like collect_apt_sources) so a non-utf8 byte cannot
lose a line; the matched lines are decoded leniently for the JSON.
"""
regex = re.compile(utils.bstr(pattern))
def collect():
# type: () -> Optional[List[str]]
if not os.path.exists(path):
return None
with open(path, 'rb') as f:
lines = f.read().splitlines()
return [line.decode('utf-8', 'replace') for line in lines if regex.search(line)]
return collect
def cmd_field(argv, as_lines=False):
# type: (List[str], bool) -> Callable[[], Any]
"""`run "<cmd>"` captured as a doctor.json field instead of a file."""
def collect():
# type: () -> Any
_, stdout, _ = run_command(argv, catch_stdout=True, catch_stderr=True)
text = (stdout or '').strip()
return text.splitlines() if as_lines else text
return collect
def collect_doctor_version():
# type: () -> str
return DOCTOR_VERSION
@utils.cached
def get_main_ip():
# type: () -> str
"""the public IP of the machine as seen by the patch server.
Cached: the main_ip and server_id fields share one HTTP roundtrip
per run.
"""
try:
response = http_utils.urlopen(utils.get_patch_server_url('myip'))
return utils.nstr(response.read()).strip()
except Exception:
return 'NA'
def collect_server_id():
# type: () -> str
server_id = serverid.get_serverid()
if not server_id:
# like kcdoctor.sh: fall back to the main IP with dots replaced
server_id = get_main_ip().replace('.', '_')
return utils.nstr(server_id)
def collect_kernel_id():
# type: () -> str
# kcdoctor.sh runs `sha1sum /proc/version`; get_kernel_hash() is the
# same digest without the trailing file name
return kcare.get_kernel_hash()
def collect_uname():
# type: () -> Dict[str, str]
result = {} # type: Dict[str, str]
for flag in ('a', 'r', 'm', 'p', 'o'):
_, stdout, _ = run_command(['uname', '-' + flag], catch_stdout=True, catch_stderr=True)
result[flag] = (stdout or '').strip()
return result
def collect_grub2_entries():
# type: () -> Optional[List[str]]
"""`grep vmlinuz /boot/grub2/grub.cfg | sed 's/root=.*//'`"""
if not os.path.exists(GRUB2_CFG):
return None
with open(GRUB2_CFG, 'rb') as f:
text = f.read().decode('utf-8', 'replace')
return [re.sub(r'root=.*', '', line) for line in text.splitlines() if 'vmlinuz' in line]
def collect_yum_repos():
# type: () -> List[str]
# `echo /etc/yum.repos.d/*`: the repo files present (empty when none)
return sorted(glob.glob(YUM_REPOS_GLOB))
def collect_control_panel():
# type: () -> Dict[str, Any]
"""presence-only port of the kcdoctor.sh detect_cp() probes"""
panels = [name for name, probe in platform_utils.CONTROL_PANEL_PROBES if probe()]
return {'cp': panels, 'softaculous': platform_utils.has_softaculous()}
# --- bulk collectors (one archive file each) ---
def collect_ipcs(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
"""`ipcs -m | sed -e s/-/=/g`"""
_, stdout, _ = run_command(['ipcs', '-m'], catch_stdout=True, catch_stderr=True)
data_package.add_file(arcname, data_bytes=utils.bstr((stdout or '').replace('-', '='), encoding='utf-8'))
def collect_dpkg_install_log(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
"""`grep ' install ' /var/log/dpkg.log`"""
if not os.path.exists(DPKG_LOG):
data_package.log_error('file not found: {0}'.format(DPKG_LOG))
return
with open(DPKG_LOG, 'rb') as f:
lines = [line for line in f.read().splitlines() if b' install ' in line]
data_package.add_file(arcname, data_bytes=b'\n'.join(lines) + b'\n')
def collect_packages(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
# same package list commands as anomaly.prepare_kernel_anomaly_report
if os.path.exists('/usr/bin/rpm') or os.path.exists('/bin/rpm'):
packages_cmd = r'rpm -q -a --queryformat="%{N}|%{V}-%{R}|%{arch}|%{INSTALLTIME:date}\n"'
elif os.path.exists('/usr/bin/dpkg'):
packages_cmd = r'/usr/bin/dpkg-query -W -f "${binary:Package}|${Version}|${Architecture}\n"'
# an unreadable dpkg.log must not lose the package list below
# (partial results, like the other multi-source collectors)
try:
collect_dpkg_install_log(data_package, 'dpkg.log')
except Exception as e:
data_package.log_error('failed to collect dpkg.log: {0}'.format(e))
else:
packages_cmd = 'echo "unknown package manager"'
data_package.add_stdout(arcname, packages_cmd)
def collect_apt_sources(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
"""`grep -rE '^(deb|URIs)' /etc/apt/sources.list*`"""
regex = re.compile(br'^(deb|URIs)')
files = [] # type: List[str]
for path in sorted(glob.glob(APT_SOURCES_GLOB)):
if os.path.isdir(path):
for root, _, names in os.walk(path):
files.extend(os.path.join(root, name) for name in sorted(names))
else:
files.append(path)
# like the shell `grep`: report the no-match case in errors.log
# instead of storing an empty entry
if not files:
data_package.log_error('file not found: {0}'.format(APT_SOURCES_GLOB))
return
lines = [] # type: List[bytes]
for path in files:
# an unreadable file must not lose the lines of the others
try:
with open(path, 'rb') as f:
content = f.read()
except Exception as e:
data_package.log_error('failed to read {0}: {1}'.format(path, e))
continue
prefix = utils.bstr('{0}:'.format(path), encoding='utf-8')
lines.extend(prefix + line for line in content.splitlines() if regex.search(line))
data_package.add_file(arcname, data_bytes=b'\n'.join(lines) + b'\n')
def collect_libcare_logs(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
"""`tail -n10000 /var/log/libcare/*.log`"""
paths = sorted(glob.glob(LIBCARE_LOGS_GLOB))
# like the shell `tail`: report the no-match case in errors.log
# instead of storing an empty entry
if not paths:
data_package.log_error('file not found: {0}'.format(LIBCARE_LOGS_GLOB))
return
chunks = [] # type: List[bytes]
for path in paths:
# an unreadable log must not lose the tails of the others
try:
with open(path, 'rb') as f:
lines = f.read().splitlines()[-LOG_TAIL_LINES:]
except Exception as e:
data_package.log_error('failed to read {0}: {1}'.format(path, e))
continue
header = utils.bstr('==> {0} <==\n'.format(path), encoding='utf-8')
chunks.append(header + b'\n'.join(lines))
data_package.add_file(arcname, data_bytes=b'\n'.join(chunks) + b'\n')
def collect_crash_dumps(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
"""crashreporter artifacts from /var/cache/kcare/dumps"""
if not os.path.isdir(config.KDUMPS_DIR):
return
data_package.add_stdout(arcname, 'ls -lR {0}'.format(config.KDUMPS_DIR))
for root, _, names in os.walk(config.KDUMPS_DIR):
for name in sorted(names):
if any(fnmatch.fnmatch(name, pattern) for pattern in CRASH_DUMP_PATTERNS):
path = os.path.join(root, name)
# keep the path relative to KDUMPS_DIR in the arcname so
# same-named dumps from different subdirectories do not
# overwrite each other in the archive
rel_path = os.path.relpath(path, config.KDUMPS_DIR)
data_package.add_file('crashreporter/{0}'.format(rel_path), src_path=path)
def collect_kdump(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
# kcdoctor.sh gates kdump collection on `[ -e /etc/kdump.conf ]`; without
# it get_kdump_root() defaults to /var/crash, so an unconfigured host with
# a stale crash dir would otherwise be collected.
if not os.path.exists(KDUMP_CONF):
return
kdump_root = kcare.get_kdump_root()
if not os.path.isdir(kdump_root):
return
data_package.add_stdout(arcname, 'ls -lR {0}'.format(kdump_root))
for evt in sorted(glob.glob(os.path.join(kdump_root, '*'))):
# only crash event directories can hold a vmcore-dmesg.txt;
# skip regular files to keep errors.log free of noise
if not os.path.isdir(evt):
continue
data_package.add_file(
'kdump/{0}/vmcore-dmesg.txt'.format(os.path.basename(evt)),
src_path=os.path.join(evt, 'vmcore-dmesg.txt'),
)
def collect_sysfs_config(data_package, arcname):
# type: (delivery_kit.DataPackage, str) -> None
"""tunables gathered by the kcdoctor.sh get_sysfs_info() `find /sys` calls"""
chunks = []
for root, dirs, names in os.walk(SYSFS_ROOT):
if root.startswith(SYSFS_EXCLUDE_PREFIXES):
dirs[:] = [] # do not descend into excluded subtrees
continue
for name in sorted(names):
if any(fnmatch.fnmatch(name, pattern) for pattern in SYSFS_NAME_PATTERNS):
path = os.path.join(root, name)
try:
with open(path, 'rb') as f:
content = f.read()
except Exception:
content = b''
chunks.append(utils.bstr('cat {0}\n'.format(path)) + content + b'\n')
data_package.add_file(arcname, data_bytes=b''.join(chunks))
# the full kcdoctor.sh (1.0-8) vitals set, in the script order; see
# docs/features/KPT-5984-doctor-v2/parity-inventory.md for the mapping
# scalar/short items, consolidated into doctor.json (one field each)
FIELD_SPECS = [
FieldSpec('doctor_version', collect_doctor_version),
FieldSpec('virt_what', cmd_field(['/usr/libexec/kcare/virt-what'], as_lines=True)),
FieldSpec('main_ip', get_main_ip),
FieldSpec('server_id', collect_server_id),
FieldSpec('kernel_id', collect_kernel_id),
FieldSpec('date', cmd_field(['date'])),
FieldSpec('uname', collect_uname),
FieldSpec('redhat_release', read_field('/etc/redhat-release')),
FieldSpec('debian_version', read_field('/etc/debian_version')),
FieldSpec('os_release', read_field('/etc/os-release')),
FieldSpec('issue', read_field('/etc/issue')),
FieldSpec('sysconfig_kernel', read_field('/etc/sysconfig/kernel')),
FieldSpec('proc_uptime', read_field('/proc/uptime')),
FieldSpec('proc_loadavg', read_field('/proc/loadavg')),
FieldSpec('proc_cmdline', read_field('/proc/cmdline')),
FieldSpec('proc_version', read_field('/proc/version')),
FieldSpec('default_grub', grep_field('/etc/default/grub', 'DEFAULT')),
FieldSpec('grub2_entries', collect_grub2_entries),
FieldSpec('sshd_port', grep_field('/etc/ssh/sshd_config', 'Port')),
FieldSpec('yum_repos_d', collect_yum_repos),
FieldSpec('control_panel', collect_control_panel),
]
# bulk items, one archive file each
FILE_SPECS = [
run_spec('syslog', 'tail -n{0} /var/log/syslog'.format(LOG_TAIL_LINES)),
run_spec('dmesg', 'dmesg'),
run_spec('messages', 'tail -n{0} /var/log/messages'.format(LOG_TAIL_LINES)),
run_spec('ls_var_cache_kcare', 'ls -lR /var/cache/kcare/'),
dump_spec('kcare.conf', '/etc/sysconfig/kcare/kcare.conf'),
run_spec('cpuinfo', 'cat /proc/cpuinfo'),
proc_dump_spec('proc_vmstat', '/proc/vmstat'),
proc_dump_spec('proc_devices', '/proc/devices'),
proc_dump_spec('proc_diskstats', '/proc/diskstats'),
proc_dump_spec('proc_mdstat', '/proc/mdstat'),
proc_dump_spec('proc_meminfo', '/proc/meminfo'),
proc_dump_spec('proc_swaps', '/proc/swaps'),
proc_dump_spec('proc_filesystems', '/proc/filesystems'),
proc_dump_spec('proc_mounts', '/proc/mounts'),
proc_dump_spec('proc_interrupts', '/proc/interrupts'),
dump_spec('grub.conf', '/boot/grub/grub.conf'),
proc_dump_spec('proc_modules', '/proc/modules'),
dump_spec('grub2.cfg', GRUB2_CFG),
proc_dump_spec('proc_zoneinfo', '/proc/zoneinfo'),
run_spec('ls_boot_configs', 'ls /etc/grub.conf /boot/grub/grub.conf /boot/grub/menu.lst'),
run_spec('ls_boot', 'ls -l /boot'),
run_spec('printenv', 'printenv'),
run_spec('dmidecode', 'dmidecode'),
callable_spec('ipcs', collect_ipcs),
run_spec('sysctl', 'sysctl -a'),
dump_spec('sysctl.conf', '/etc/sysctl.conf'),
callable_spec('packages.list', collect_packages),
run_spec('lspci', 'lspci -vv'),
run_spec('dpkg_l', 'dpkg -l'),
callable_spec('apt_sources', collect_apt_sources),
dump_spec('yum.conf', '/etc/yum.conf'),
run_spec('yum_repolist', 'yum repolist'),
callable_spec('libcare_logs', collect_libcare_logs),
run_spec('kcarectl.log', 'tail -n{0} /var/log/kcarectl.log'.format(LOG_TAIL_LINES)),
dump_spec('kdump.conf', KDUMP_CONF),
dump_spec('kcare-cron', '/etc/cron.d/kcare-cron'),
callable_spec('crashreporter/ls', collect_crash_dumps),
callable_spec('kdump/ls', collect_kdump),
run_spec('aa_status', 'aa-status'),
run_spec('sestatus', 'sestatus'),
run_spec('kprobes_list', 'cat /sys/kernel/debug/kprobes/list'),
callable_spec('sysfs_config', collect_sysfs_config),
run_spec('lsblk', 'lsblk -f'),
run_spec('df', 'df -h'),
]
@utils.catch_errors(logger=log_utils.logwarn)
def prepare_doctor_report():
# type: () -> Optional[KcarectlDoctorPackage]
# None when collection fails (catch_errors): KPT-6064/KPT-6065
# callers must handle it
data_package = KcarectlDoctorPackage()
with data_package:
report = {} # type: Dict[str, Any]
for field in FIELD_SPECS:
field.apply(data_package, report)
data_package.add_json('doctor.json', report)
for spec in FILE_SPECS:
spec.apply(data_package)
return data_package
def send_doctor_report(fallback, force_fallback=False):
# type: (Callable[[], None], bool) -> None
"""Collect and upload the doctor report via the generic uploads proxy.
PUTs the archive to ``/upload/kcarectl-doctor/`` -- the same modern
pipeline as ``kernel-anomaly`` -- which ePortal proxies transparently
to the patch server, so no ePortal change is required. Any 2xx reply
is success, including the ``REDUCED_REPORT`` ``204`` where the proxy
intentionally drops the report (``send()`` returns without raising, so
we do not fall back).
Falls back to the legacy ``kcdoctor.sh`` flow (``fallback``) when:
* ``force_fallback`` (the ``--doctor --fallback`` CLI flag) or
``config.FORCE_DOCTOR_FALLBACK`` is set -- the rollout kill-switch.
``--fallback`` is the one-off, per-invocation form; the config knob
is settable per host in kcare.conf or pushed fleet-wide via the
``KC-Flag-Force-Doctor-Fallback`` feature flag. Either forces the
legacy flow on every host, registered or not, before the new path
is even considered;
* the host is not registered, i.e. there is no auth string (OQ-4: the
proxy upload requires auth, so the new path is not even attempted);
* report collection failed (``prepare_doctor_report`` returned None);
* the new route is unavailable on an older patch server / ePortal --
a 404 / 405 / 403, surfaced as ``HTTPError``, or a connection-level
failure. ``send()`` drives ``http_utils.upload_file`` (raw
``httplib``, not ``urllib``), so transport failures surface as an
``OSError`` subclass -- ``socket.timeout``, ``ConnectionRefused``,
``socket.gaierror`` (DNS), ``ssl.SSLError`` -- or an
``httplib.HTTPException``, *not* ``URLError``; all of these mean the
new path is unusable. ``upload_file`` raises only on status >= 400,
so a redirect is not treated as a failure; the proxy and patch
server never redirect uploads.
:param fallback: legacy ``kcdoctor.sh`` runner, injected by the caller
(KPT-6065) to keep this module free of an ``__init__`` import cycle.
:param force_fallback: when True (``--doctor --fallback``), skip the v2
path and run ``fallback`` directly; the per-invocation form of
``config.FORCE_DOCTOR_FALLBACK``.
"""
if force_fallback or config.FORCE_DOCTOR_FALLBACK:
src = '--fallback requested' if force_fallback else 'FORCE_DOCTOR_FALLBACK is set'
log_utils.loginfo(
'doctor: {0}; using the legacy kcdoctor.sh flow'.format(src),
print_msg=False,
)
fallback()
return
if auth.get_http_auth_string() is None:
log_utils.loginfo(
'doctor: host is not registered; using the legacy kcdoctor.sh flow',
print_msg=False,
)
fallback()
return
data_package = prepare_doctor_report()
if data_package is None:
log_utils.logwarn('doctor: report collection failed; using the legacy kcdoctor.sh flow')
fallback()
return
try:
upload_name = data_package.send()
log_utils.loginfo('doctor: report uploaded as {0}'.format(upload_name), print_msg=False)
# surface the uploaded filename to the user, the same way the eportal
# --doctor does (kc.eportal make_doctor_report -> utils.print_wrapper)
# so support can locate the report; the legacy kcdoctor.sh fallback
# prints its own "Key:" line instead. print_wrapper (not loginfo) so
# the line is shown unconditionally -- loginfo is gated by PRINT_LEVEL
# and would be dropped under --quiet/--auto-update, whereas the legacy
# kcdoctor.sh echo is not.
utils.print_wrapper('Please provide this filename to support: {0}'.format(upload_name))
except (HTTPError, URLError, httplib.HTTPException, socket.error) as err:
# upload_file drives httplib directly, so a transport failure
# surfaces as socket.error (== OSError on py3: timeout, connection
# refused, DNS, TLS) or httplib.HTTPException -- not URLError. An
# HTTPError (route absent: 404 / 405 / 403) means the same thing:
# the new path is unusable, so fall back to the legacy flow.
log_utils.logwarn('doctor: report upload unavailable ({0}); using the legacy kcdoctor.sh flow'.format(err))
fallback()
finally:
data_package.remove_archive()