From 6b76a586fc05e7f5e66ebfd9e48e7d8cfa5d163e Mon Sep 17 00:00:00 2001 From: mprovenc Date: Wed, 10 Mar 2021 08:15:20 -0500 Subject: [PATCH 01/29] Initial kernel_report module commit Basic functionality of kernel settings report module --- library/kernel_report.py | 55 ++++++++++++++++++++++++++++++++++++++++ utils/procsysfswalk.sh | 5 ++++ 2 files changed, 60 insertions(+) create mode 100644 library/kernel_report.py create mode 100644 utils/procsysfswalk.sh diff --git a/library/kernel_report.py b/library/kernel_report.py new file mode 100644 index 00000000..1ed441ec --- /dev/null +++ b/library/kernel_report.py @@ -0,0 +1,55 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2021, Mary Provencher +# SPDX-License-Identifier: GPL-2.0-or-later +# +""" Generate kernel settings facts for a system """ + +import subprocess as sp +from ansible.module_utils.basic import AnsibleModule + +def add_dict_level(dct, lvls, val): + """ + Function that recursively adds levels to a dictionary. Takes an existing dictionary, a list of nested levels, + and a value to be assigned to the deepest level. + """ + key = lvls[0].rstrip() + if len(lvls) == 1: + dct[key] = val.lstrip() + else: + dct[key] = add_dict_level(dct[key] if key in dct else {}, lvls[1:], val) + return dct + +def run_module(): + module_args = dict() + + result = dict( + changed=False, + ansible_facts=dict(), + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + if module.check_mode: + module.exit_json(**result) + + procfs_output = sp.run(['sh', 'utils/procsysfswalk.sh'], stdout=sp.PIPE).stdout.decode("utf-8")[:-2] + facts_as_dict = dict() + + for substring in procfs_output.split('\n'): + keyval = substring.split('=') + facts_as_dict = add_dict_level(facts_as_dict, keyval[0].split('/')[1:], keyval[1]) + + result['ansible_facts'] = facts_as_dict + + module.exit_json(**result) + +def main(): + run_module() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/utils/procsysfswalk.sh b/utils/procsysfswalk.sh new file mode 100644 index 00000000..cc4b3f41 --- /dev/null +++ b/utils/procsysfswalk.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for i in $(find '/proc/sys' -type f -perm -600) +do + echo "${i:9}" = $(grep -v '^#' $i 2>&-) +done \ No newline at end of file From 171695f7791d0ab32fcf1264f330ddd993dc53c0 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Wed, 10 Mar 2021 08:37:08 -0500 Subject: [PATCH 02/29] Rename shell utility for consistency --- library/kernel_report.py | 2 +- utils/{procsysfswalk.sh => sysctlwalk.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename utils/{procsysfswalk.sh => sysctlwalk.sh} (100%) diff --git a/library/kernel_report.py b/library/kernel_report.py index 1ed441ec..76c7cc73 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -37,7 +37,7 @@ def run_module(): if module.check_mode: module.exit_json(**result) - procfs_output = sp.run(['sh', 'utils/procsysfswalk.sh'], stdout=sp.PIPE).stdout.decode("utf-8")[:-2] + procfs_output = sp.run(['sh', 'utils/sysctlwalk.sh'], stdout=sp.PIPE).stdout.decode("utf-8")[:-2] facts_as_dict = dict() for substring in procfs_output.split('\n'): diff --git a/utils/procsysfswalk.sh b/utils/sysctlwalk.sh similarity index 100% rename from utils/procsysfswalk.sh rename to utils/sysctlwalk.sh From 74c1ea19345f4e81b3541be93d7afb7df13954d4 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Wed, 10 Mar 2021 08:53:47 -0500 Subject: [PATCH 03/29] change procfs to sysctl --- library/kernel_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/kernel_report.py b/library/kernel_report.py index 76c7cc73..6d422647 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -37,10 +37,10 @@ def run_module(): if module.check_mode: module.exit_json(**result) - procfs_output = sp.run(['sh', 'utils/sysctlwalk.sh'], stdout=sp.PIPE).stdout.decode("utf-8")[:-2] + sysctl_output = sp.run(['sh', 'utils/sysctlwalk.sh'], stdout=sp.PIPE).stdout.decode("utf-8")[:-2] facts_as_dict = dict() - for substring in procfs_output.split('\n'): + for substring in sysctl_output.split('\n'): keyval = substring.split('=') facts_as_dict = add_dict_level(facts_as_dict, keyval[0].split('/')[1:], keyval[1]) From 37bd16b7b7679dd28c8bf2b838dbae0ad33bf83c Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 11 Mar 2021 10:09:33 -0500 Subject: [PATCH 04/29] Change shell utility file to python functions Move everything to python functionality instead of using shell scripts, change the format of the output to be more consistent with kernel_settings role input, filter out some known unstable values --- library/kernel_report.py | 44 +++++++++++++++++++++++----------------- utils/sysctlwalk.sh | 5 ----- 2 files changed, 25 insertions(+), 24 deletions(-) delete mode 100644 utils/sysctlwalk.sh diff --git a/library/kernel_report.py b/library/kernel_report.py index 6d422647..600af795 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -6,20 +6,33 @@ # """ Generate kernel settings facts for a system """ +import os import subprocess as sp from ansible.module_utils.basic import AnsibleModule -def add_dict_level(dct, lvls, val): - """ - Function that recursively adds levels to a dictionary. Takes an existing dictionary, a list of nested levels, - and a value to be assigned to the deepest level. - """ - key = lvls[0].rstrip() - if len(lvls) == 1: - dct[key] = val.lstrip() - else: - dct[key] = add_dict_level(dct[key] if key in dct else {}, lvls[1:], val) - return dct +UNSTABLE_SYSCTL_FIELDS = ('kernel.hostname', 'kernel.domainname', 'dev', 'kernel.ns_last_pid', 'net.netfilter.nf_conntrack_events') +SYSCTL_DIR = '/proc/sys' + +def file_get_contents(filename): + with open(filename) as f: + return f.read().rstrip() + +def sysctl_walk(): + result = [] + for dirpath, dirs, files in os.walk(SYSCTL_DIR): + if files: + for file in files: + setting_path = dirpath + "/" + file + if(int(oct(os.stat(setting_path).st_mode)[-3:]) >= 600): + formatted_setting = str(setting_path[10:]).replace("/",".") + if formatted_setting not in UNSTABLE_SYSCTL_FIELDS: + try: + val = file_get_contents(setting_path) + result.append({'name': formatted_setting, "value": val}) + except OSError as e: + # read errors occur on some of the 'stable_secret' files + pass + return result def run_module(): module_args = dict() @@ -37,14 +50,7 @@ def run_module(): if module.check_mode: module.exit_json(**result) - sysctl_output = sp.run(['sh', 'utils/sysctlwalk.sh'], stdout=sp.PIPE).stdout.decode("utf-8")[:-2] - facts_as_dict = dict() - - for substring in sysctl_output.split('\n'): - keyval = substring.split('=') - facts_as_dict = add_dict_level(facts_as_dict, keyval[0].split('/')[1:], keyval[1]) - - result['ansible_facts'] = facts_as_dict + result['ansible_facts'] = {"sysctl":sysctl_walk()} module.exit_json(**result) diff --git a/utils/sysctlwalk.sh b/utils/sysctlwalk.sh deleted file mode 100644 index cc4b3f41..00000000 --- a/utils/sysctlwalk.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -for i in $(find '/proc/sys' -type f -perm -600) -do - echo "${i:9}" = $(grep -v '^#' $i 2>&-) -done \ No newline at end of file From 141d0cb6a6652082cb0658e07a4737f1188df1e0 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 11 Mar 2021 11:41:33 -0500 Subject: [PATCH 05/29] Add sysfs settings to kernel_report module Also convert unstable settings list to regex for more flexibility --- library/kernel_report.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/library/kernel_report.py b/library/kernel_report.py index 600af795..7c213314 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -7,25 +7,29 @@ """ Generate kernel settings facts for a system """ import os +import re import subprocess as sp from ansible.module_utils.basic import AnsibleModule -UNSTABLE_SYSCTL_FIELDS = ('kernel.hostname', 'kernel.domainname', 'dev', 'kernel.ns_last_pid', 'net.netfilter.nf_conntrack_events') +UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events'] +UNSTABLE_SYSFS_FIELDS = ['kernel\.debug', 'devices'] SYSCTL_DIR = '/proc/sys' +SYSFS_DIR = '/sys' def file_get_contents(filename): with open(filename) as f: return f.read().rstrip() -def sysctl_walk(): +def settings_walk(dir, unstable): result = [] - for dirpath, dirs, files in os.walk(SYSCTL_DIR): + combined_unstable = "(" + ")|(".join(unstable) + ")" + for dirpath, dirs, files in os.walk(dir): if files: for file in files: setting_path = dirpath + "/" + file if(int(oct(os.stat(setting_path).st_mode)[-3:]) >= 600): - formatted_setting = str(setting_path[10:]).replace("/",".") - if formatted_setting not in UNSTABLE_SYSCTL_FIELDS: + formatted_setting = str(setting_path[len(dir)+1:]).replace("/",".") + if re.match(combined_unstable,formatted_setting) is None: try: val = file_get_contents(setting_path) result.append({'name': formatted_setting, "value": val}) @@ -50,7 +54,7 @@ def run_module(): if module.check_mode: module.exit_json(**result) - result['ansible_facts'] = {"sysctl":sysctl_walk()} + result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS), "sysfs":settings_walk(SYSFS_DIR, UNSTABLE_SYSFS_FIELDS)} module.exit_json(**result) From 4e9074974b1ca8c5f69e29604bf0dec10dadaedf Mon Sep 17 00:00:00 2001 From: mprovenc Date: Mon, 15 Mar 2021 14:18:24 -0400 Subject: [PATCH 06/29] Create test to apply template settings en masse Simple two host test to apply kernel settings output from kernel_report module to another machine --- library/kernel_report.py | 6 ++++-- tests/tests_apply_kernel_report_settings.yml | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/tests_apply_kernel_report_settings.yml diff --git a/library/kernel_report.py b/library/kernel_report.py index 7c213314..9f1d4972 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -27,12 +27,13 @@ def settings_walk(dir, unstable): if files: for file in files: setting_path = dirpath + "/" + file - if(int(oct(os.stat(setting_path).st_mode)[-3:]) >= 600): + if(int(oct(os.stat(setting_path).st_mode)[-3:]) >= 644): formatted_setting = str(setting_path[len(dir)+1:]).replace("/",".") if re.match(combined_unstable,formatted_setting) is None: try: val = file_get_contents(setting_path) - result.append({'name': formatted_setting, "value": val}) + if val: + result.append({'name': formatted_setting, "value": val}) except OSError as e: # read errors occur on some of the 'stable_secret' files pass @@ -54,6 +55,7 @@ def run_module(): if module.check_mode: module.exit_json(**result) + # result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS)} result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS), "sysfs":settings_walk(SYSFS_DIR, UNSTABLE_SYSFS_FIELDS)} module.exit_json(**result) diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml new file mode 100644 index 00000000..8aa27916 --- /dev/null +++ b/tests/tests_apply_kernel_report_settings.yml @@ -0,0 +1,19 @@ +- hosts: all + + roles: + - role: linux-system-roles.kernel_settings + when: false + + tasks: + - name: get and store config values from template machine + kernel_report: + register: template_settings + when: inventory_hostname == 'template_machine' + no_log: true + + - name: apply kernel_settings to target machine(s) + include_role: + name: linux-system-roles.kernel_settings + vars: + kernel_settings_sysctl: "{{ hostvars['template_machine'].template_settings.ansible_facts.sysctl }}" + when: inventory_hostname == 'target_machine' From 2ff7adcec6ceabb87091126a7dc52e88c5a8e3ec Mon Sep 17 00:00:00 2001 From: mprovenc Date: Mon, 15 Mar 2021 14:32:37 -0400 Subject: [PATCH 07/29] fix yamllint errors --- tests/tests_apply_kernel_report_settings.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index 8aa27916..7ce53e38 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -15,5 +15,7 @@ include_role: name: linux-system-roles.kernel_settings vars: - kernel_settings_sysctl: "{{ hostvars['template_machine'].template_settings.ansible_facts.sysctl }}" + kernel_settings_sysctl: >- + {{ hostvars['template_machine']. + template_settings.ansible_facts.sysctl }} when: inventory_hostname == 'target_machine' From 3c5c29c988183bafdc175b21aa85cec3c75bac0d Mon Sep 17 00:00:00 2001 From: mprovenc Date: Tue, 16 Mar 2021 14:39:11 -0400 Subject: [PATCH 08/29] Make compatible with local qemu testing Assumes that two images will be specified as TEST_SUBJECTS: One CentOS image and one Fedora image --- tests/tests_apply_kernel_report_settings.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index 7ce53e38..cb40c566 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -8,7 +8,7 @@ - name: get and store config values from template machine kernel_report: register: template_settings - when: inventory_hostname == 'template_machine' + when: ansible_distribution == 'CentOS' no_log: true - name: apply kernel_settings to target machine(s) @@ -16,6 +16,8 @@ name: linux-system-roles.kernel_settings vars: kernel_settings_sysctl: >- - {{ hostvars['template_machine']. - template_settings.ansible_facts.sysctl }} - when: inventory_hostname == 'target_machine' + {{ hostvars[item].template_settings.ansible_facts.sysctl }} + loop: "{{ groups['all'] }}" + when: + - hostvars[item].ansible_distribution == 'CentOS' + - ansible_distribution == 'Fedora' From 27414b7594d23958d72542b5a3d04c664d267f45 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 1 Apr 2021 09:13:40 -0400 Subject: [PATCH 09/29] Add drop_caches as unstable sysctl field --- library/kernel_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/kernel_report.py b/library/kernel_report.py index 9f1d4972..8182286a 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -11,7 +11,7 @@ import subprocess as sp from ansible.module_utils.basic import AnsibleModule -UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events'] +UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events', 'vm\.drop_caches'] UNSTABLE_SYSFS_FIELDS = ['kernel\.debug', 'devices'] SYSCTL_DIR = '/proc/sys' SYSFS_DIR = '/sys' From e8e65081f12300de17636c5a54df6176251ca681 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Mon, 5 Apr 2021 17:41:30 -0400 Subject: [PATCH 10/29] Add support for more tuned parameters Taken from the tuned documentation and existing profiles, support added for more parameters which the kernel_settings role should allow users to change --- defaults/main.yml | 35 ++++++++++ library/kernel_settings.py | 8 +++ tasks/main.yml | 120 ++++++++++++++++++++++++++++++++ tests/tests_change_settings.yml | 16 +++++ 4 files changed, 179 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index a8ddc262..7e02b99b 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -38,6 +38,41 @@ kernel_settings_transparent_hugepages: null # value. The actual supported values may be different depending on your OS. kernel_settings_transparent_hugepages_defrag: null +# cpu settings +kernel_settings_cpu_governor: null +kernel_settings_cpu_min_perf_pct: null +kernel_settings_cpu_max_perf_pct: null +kernel_settings_sampling_down_factor: null +kernel_settings_no_turbo: null + +# disk settings +kernel_settings_disk_elevator: null +kernel_settings_disk_read_ahead_kb: null +kernel_settings_disk_scheduler_quantum: null + +# selinux settings +kernel_settings_avc_cache_threshold: null + +# net settings +kernel_settings_nf_conntrack_hashsize: null + +# audio settings +kernel_settings_audio_timeout: null +kernel_settings_audio_reset_controller: null + +# scsi_host settings +kernel_settings_scsi_host_alpm: null + +# eeepc_she settings +kernel_settings_eeepc_she_powersave: null +kernel_settings_eeepc_she_normal: null + +# video settings +kernel_settings_video_radeon_powersave: null + +# usb settings +kernel_settings_usb_autosuspend: null + # If purge is true, completely wipe out whatever the current settings # are and replace them with kernel_settings_parameters kernel_settings_purge: false diff --git a/library/kernel_settings.py b/library/kernel_settings.py index 193033a5..ba7458a2 100644 --- a/library/kernel_settings.py +++ b/library/kernel_settings.py @@ -306,6 +306,14 @@ def get_supported_tuned_plugin_names(): "sysfs", "systemd", "vm", + "cpu", + "disk", + "net", + "audio", + "scsi_host", + "eeepc_she", + "video", + "usb", ] diff --git a/tasks/main.yml b/tasks/main.yml index c54138e8..6498bab4 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -41,6 +41,126 @@ - name: Apply kernel settings kernel_settings: + cpu: + - name: "{{ 'governor' if kernel_settings_cpu_governor + else none }}" + value: "{{ kernel_settings_cpu_governor + if kernel_settings_cpu_governor != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_cpu_governor == + __kernel_settings_state_absent else none }}" + - name: "{{ 'min_perf_pct' if kernel_settings_cpu_min_perf_pct + else none }}" + value: "{{ kernel_settings_cpu_min_perf_pct + if kernel_settings_cpu_min_perf_pct != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_cpu_min_perf_pct == + __kernel_settings_state_absent else none }}" + - name: "{{ 'max_perf_pct' if kernel_settings_cpu_max_perf_pct + else none }}" + value: "{{ kernel_settings_cpu_max_perf_pct + if kernel_settings_cpu_max_perf_pct != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_cpu_max_perf_pct == + __kernel_settings_state_absent else none }}" + - name: "{{ 'sampling_down_factor' if kernel_settings_sampling_down_factor + else none }}" + value: "{{ kernel_settings_sampling_down_factor + if kernel_settings_sampling_down_factor != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_sampling_down_factor == + __kernel_settings_state_absent else none }}" + - name: "{{ 'no_turbo' if kernel_settings_no_turbo + else none }}" + value: "{{ kernel_settings_no_turbo + if kernel_settings_no_turbo != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_no_turbo == + __kernel_settings_state_absent else none }}" + disk: + - name: "{{ 'elevator' if kernel_settings_disk_elevator + else none }}" + value: "{{ kernel_settings_disk_elevator + if kernel_settings_disk_elevator != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_disk_elevator == + __kernel_settings_state_absent else none }}" + - name: "{{ 'readahead' if kernel_settings_disk_read_ahead_kb + else none }}" + value: "{{ kernel_settings_disk_read_ahead_kb + if kernel_settings_disk_read_ahead_kb != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_disk_read_ahead_kb == + __kernel_settings_state_absent else none }}" + - name: "{{ 'scheduler_quantum' if kernel_settings_disk_scheduler_quantum + else none }}" + value: "{{ kernel_settings_disk_scheduler_quantum + if kernel_settings_disk_scheduler_quantum != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_disk_scheduler_quantum == + __kernel_settings_state_absent else none }}" + net: + - name: "{{ 'nf_conntrack_hashsize' if kernel_settings_nf_conntrack_hashsize + else none }}" + value: "{{ kernel_settings_nf_conntrack_hashsize + if kernel_settings_nf_conntrack_hashsize != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_nf_conntrack_hashsize == + __kernel_settings_state_absent else none }}" + audio: + - name: "{{ 'timeout' if kernel_settings_audio_timeout + else none }}" + value: "{{ kernel_settings_audio_timeout + if kernel_settings_audio_timeout != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_audio_timeout == + __kernel_settings_state_absent else none }}" + - name: "{{ 'reset_controller' if kernel_settings_audio_reset_controller + else none }}" + value: "{{ kernel_settings_audio_reset_controller + if kernel_settings_audio_reset_controller != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_audio_reset_controller == + __kernel_settings_state_absent else none }}" + scsi_host: + - name: "{{ 'alpm' if kernel_settings_scsi_host_alpm + else none }}" + value: "{{ kernel_settings_scsi_host_alpm + if kernel_settings_scsi_host_alpm != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_scsi_host_alpm == + __kernel_settings_state_absent else none }}" + eeepc_she: + - name: "{{ 'she_powersave' if kernel_settings_eeepc_she_powersave + else none }}" + value: "{{ kernel_settings_eeepc_she_powersave + if kernel_settings_eeepc_she_powersave != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_eeepc_she_powersave == + __kernel_settings_state_absent else none }}" + - name: "{{ 'she_normal' if kernel_settings_eeepc_she_normal + else none }}" + value: "{{ kernel_settings_eeepc_she_normal + if kernel_settings_eeepc_she_normal != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_eeepc_she_normal == + __kernel_settings_state_absent else none }}" + video: + - name: "{{ 'radeon_powersave' if kernel_settings_video_radeon_powersave + else none }}" + value: "{{ kernel_settings_video_radeon_powersave + if kernel_settings_video_radeon_powersave != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_video_radeon_powersave == + __kernel_settings_state_absent else none }}" + usb: + - name: "{{ 'autosuspend' if kernel_settings_usb_autosuspend + else none }}" + value: "{{ kernel_settings_usb_autosuspend + if kernel_settings_usb_autosuspend != + __kernel_settings_state_absent else none }}" + state: "{{ 'absent' if kernel_settings_usb_autosuspend == + __kernel_settings_state_absent else none }}" sysctl: "{{ kernel_settings_sysctl if kernel_settings_sysctl else omit }}" sysfs: "{{ kernel_settings_sysfs if kernel_settings_sysfs else omit }}" systemd: diff --git a/tests/tests_change_settings.yml b/tests/tests_change_settings.yml index 307f17be..70bb269c 100644 --- a/tests/tests_change_settings.yml +++ b/tests/tests_change_settings.yml @@ -35,6 +35,22 @@ kernel_settings_sysfs: - name: /sys/class/net/lo/mtu value: 65000 + kernel_settings_cpu_governor: "conservative|powersave" + kernel_settings_cpu_min_perf_pct: 20 + kernel_settings_cpu_max_perf_pct: 99 + kernel_settings_sampling_down_factor: 98 + kernel_settings_no_turbo: 1 + kernel_settings_disk_elevator: "bfq" + kernel_settings_disk_read_ahead_kb: 256 + kernel_settings_disk_scheduler_quantum: 64 + kernel_settings_nf_conntrack_hashsize: 1048576 + kernel_settings_audio_timeout: 10 + kernel_settings_audio_reset_controller: 1 + kernel_settings_scsi_host_alpm: "min_power" + kernel_settings_eeepc_she_powersave: 2 + kernel_settings_eeepc_she_normal: 1 + kernel_settings_video_radeon_powersave: "auto" + kernel_settings_usb_autosuspend: 1 - name: check sysfs after role runs command: grep -x 65000 /sys/class/net/lo/mtu From 22f6e1b3fdca302888e5e003e1a5a34bcce9f016 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 8 Apr 2021 09:29:49 -0400 Subject: [PATCH 11/29] Rewrite kernel_settings plugin with new fields --- defaults/main.yml | 6 +- library/kernel_settings.py | 981 ++++---------------------------- library/old_kernel_settings.py | 892 +++++++++++++++++++++++++++++ tasks/main.yml | 15 - tests/tests_change_settings.yml | 2 - 5 files changed, 1007 insertions(+), 889 deletions(-) create mode 100644 library/old_kernel_settings.py diff --git a/defaults/main.yml b/defaults/main.yml index 7e02b99b..79f47841 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -40,9 +40,9 @@ kernel_settings_transparent_hugepages_defrag: null # cpu settings kernel_settings_cpu_governor: null +kernel_settings_sampling_down_factor: null kernel_settings_cpu_min_perf_pct: null kernel_settings_cpu_max_perf_pct: null -kernel_settings_sampling_down_factor: null kernel_settings_no_turbo: null # disk settings @@ -63,10 +63,6 @@ kernel_settings_audio_reset_controller: null # scsi_host settings kernel_settings_scsi_host_alpm: null -# eeepc_she settings -kernel_settings_eeepc_she_powersave: null -kernel_settings_eeepc_she_normal: null - # video settings kernel_settings_video_radeon_powersave: null diff --git a/library/kernel_settings.py b/library/kernel_settings.py index ba7458a2..3be4d1d5 100644 --- a/library/kernel_settings.py +++ b/library/kernel_settings.py @@ -1,893 +1,140 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -# Copyright: (c) 2020, Rich Megginson +# Copyright: (c) 2021, Mary Provencher # SPDX-License-Identifier: GPL-2.0-or-later # -""" Manage kernel settings using tuned via a wrapper """ +""" Generate kernel settings facts for a system """ -from __future__ import absolute_import, division, print_function - -__metaclass__ = type - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - -DOCUMENTATION = """ ---- -module: kernel_settings - -short_description: Manage kernel settings using tuned via a wrapper - -version_added: "2.8" - -description: - - | - Manage kernel settings using tuned via a wrapper. The options correspond - to names of units or plugins in tuned. For example, the option C(sysctl) - corresponds to the C(sysctl) unit or plugin in tuned. Setting parameters - works mostly like it does with tuned, except that this module uses Ansible - YAML format instead of the tuned INI-style profile file format. This module - creates a special tuned profile C(kernel_settings) which will be applied by - tuned before any other profiles, allowing the user to configure tuned to - override settings made by this module. You should be aware of this if you - plan to use tuned in addition to using this module. - - HORIZONTALLINE - - | - NOTE: the options list may be incomplete - the actual options are generated - dynamically from tuned, for the current options supported by the version - of tuned, which are the tuned supported plugins. Only the most common - options are listed. See the tuned documentation for the full list and - more information. - - HORIZONTALLINE - - | - Each option takes a list or dict of settings. Each setting is a C(dict). - The C(dict) must have one of the keys C(name), C(value), C(state), or C(previous). - C(state) is used to remove settings or sections of settings. C(previous) is - used to replace all of values in a section with the given values. The only case - where an option takes a dict is when you want to remove a section completely - - then value for the section is the dict C({"state":"empty"}). - If you specify multiple settings with the same name in a section, the last one - will be used. -options: - sysctl: - description: - - | - list of sysctl settings to apply - this works mostly - like the C(sysctl) module except that C(/etc/sysctl.conf) and files - under C(/etc/sysctl.d) are not used. - required: false - sysfs: - description: - - key/value pairs of sysfs settings to apply - required: false - bootloader: - description: - - the C(cmdline) option can be used to set, add, or delete - kernel command line options. See EXAMPLES for some examples - of how to use this option. - Note that this uses the tuned implementation, which adds these - options to whatever the default bootloader command line arguments - tuned is historically used to add/delete performance related - kernel command line arguments e.g. C(spectre_v2=off). If you need - more general purpose bootloader configuration, you should use - a bootloader module/role. - type: raw - purge: - description: - - Remove the current kernel_settings, whatever they are, and force - the given settings to be the current and only settings - type: bool - default: false - -author: - - Rich Megginson (@richm) -""" - -EXAMPLES = """ -# Add or replace the sysctl `fs.file-max` parameter with the value 65535. The -# existing settings are not touched. -- name: Add or replace the sysctl `fs.file-max` parameter with the value 65535. - kernel_settings: - sysctl: - - name: fs.file-max - value: 65535 - -- name: remove the entire sysctl section - kernel_settings: - sysctl: - state: empty - -- name: remove the entire sysctl section and replace with the given values - kernel_settings: - sysctl: - - previous: replaced - - name: fs.file-max - value: 65535 - -- name: add the sysctl vm.max_mmap_regions, and disable spectre/meltdown security - kernel_settings: - sysctl: - - name: vm.max_mmap_regions - value: 262144 - sysfs: - - name: /sys/kernel/debug/x86/pti_enabled - value: 0 - - name: /sys/kernel/debug/x86/retp_enabled - value: 0 - - name: /sys/kernel/debug/x86/ibrs_enabled - value: 0 - -# replace the existing sysctl section with the specified section -# delete the /sys/kernel/debug/x86/retp_enabled setting -# completely remove the vm section -# add the bootloader cmdline arguments spectre_v2=off nopti -# remove the bootloader cmdline arguments panic splash -- name: more settings - kernel_settings: - sysctl: - - previous: replaced - - name: vm.max_mmap_regions - value: 262144 - sysfs: - - name: /sys/kernel/debug/x86/retp_enabled - state: absent - vm: - state: empty - bootloader: - - name: cmdline - value: - - name: spectre_v2 - value: off - - name: nopti - - name: panic - state: absent - - name: splash - state: absent -""" - -RETURN = """ -msg: - description: | - A short text message to say what action this module performed. - returned: always - type: str -new_profile: - description: | - This is the tuned profile in dict format, after the changes, if any, - have been applied. - returned: always - type: dict -reboot_required: - description: | - default C(false) - if true, this means a reboot of the managed host is - required in order for the changes to take effect. - returned: always - type: bool -active_profile: - description: | - This is the space delimited list of active profiles as reported - by tuned. - returned: always - type: str -""" - -import os -import logging import re -import tempfile -import shutil -import shlex -import contextlib -import atexit # for testing -import copy - -try: - import configobj - - HAS_CONFIGOBJ = True -except ImportError: - HAS_CONFIGOBJ = False -import ansible.module_utils.six as ansible_six - -# This is a bit of a mystery - bug in pylint? -# pylint: disable=import-error -import ansible.module_utils.six.moves as ansible_six_moves - +import pyudev from ansible.module_utils.basic import AnsibleModule -# see https://github.com/python/cpython/blob/master/Lib/logging/__init__.py -# for information about logging module internals - - -class TunedLogWrapper(logging.getLoggerClass()): - """This wraps the tuned logger so that we can intercept logs and handle them here""" - - def __init__(self, *args, **kwargs): - super(TunedLogWrapper, self).__init__(*args, **kwargs) - self.setLevel(logging.DEBUG) - self.logstack = [] - - def handle(self, record): - self.logstack.append(record) - - -@contextlib.contextmanager -def save_and_restore_logging_methods(): - """do not allow tuned logging to pollute global logging module or - ansible logging""" - save_logging_add_level_name = logging.addLevelName +UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events', 'vm\.drop_caches'] +UNSTABLE_SYSFS_FIELDS = ['kernel\.debug', 'devices'] +SYSCTL_DIR = '/proc/sys' +SYSFS_DIR = '/sys' - def wrapper_add_level_name(_levelval, _levelname): - """ignore tuned.logging call to logging.addLevelName""" - # print('addLevelName wrapper ignoring {} {}'.format(levelval, levelname)) - - logging.addLevelName = wrapper_add_level_name - save_logging_set_logger_class = logging.setLoggerClass - - def wrapper_set_logger_class(_clsname): - """ignore tuned.logging call to logging.setLoggerClass""" - # print('setLoggerClass wrapper ignoring {}'.format(clsname)) - - logging.setLoggerClass = wrapper_set_logger_class +def safe_file_get_contents(filename): try: - yield - finally: - logging.addLevelName = save_logging_add_level_name - logging.setLoggerClass = save_logging_set_logger_class - - -caught_import_error = None -HAS_TUNED = False -try: - with save_and_restore_logging_methods(): - import tuned.logs - HAS_TUNED = True -except ImportError as ierr: - # tuned package might not be available in check mode - so just - # note that this is missing, and do not report in check mode - caught_import_error = ierr - -if HAS_TUNED: - tuned.logs.root_logger = TunedLogWrapper(__name__) - tuned.logs.get = lambda: tuned.logs.root_logger - - import tuned.consts - import tuned.utils.global_config - import tuned.daemon - from tuned.exceptions import TunedException - - -TUNED_PROFILE = os.environ.get("TEST_PROFILE", "kernel_settings") -NOCHANGES = 0 -CHANGES = 1 -REMOVE_SECTION_VALUE = {"state": "empty"} -SECTION_TO_REPLACE = "__section_to_replace" -ERR_SECTION_MISSING_NAME = "Error: section [{0}] item is missing 'name': {1}" -ERR_NAME_NOT_VALID = "Error: section [{0}] item name [{1}] is not a valid string" -ERR_NO_VALUE_OR_STATE = ( - "Error: section [{0}] item name [{1}] must have either a 'value' or 'state'" -) -ERR_BOTH_VALUE_AND_STATE = ( - "Error: section [{0}] item name [{1}] must have only one of 'value' or 'state'" -) -ERR_STATE_ABSENT = ( - "Error: section [{0}] item name [{1}] state value must be 'absent' not [{2}]" -) -ERR_UNEXPECTED_VALUES = "Error: section [{0}] item [{1}] has unexpected values {2}" -ERR_VALUE_ITEM_NOT_DICT = ( - "Error: section [{0}] item name [{1}] value [{2}] is not a dict" -) -ERR_VALUE_ITEM_PREVIOUS = ( - "Error: section [{0}] item name [{1}] has invalid value for 'previous' [{2}]" -) -ERR_REMOVE_SECTION_VALUE = "Error: to remove the section [{0}] specify the value {1}" -ERR_ITEM_NOT_DICT = "Error: section [{0}] item value [{1}] is not a dict" -ERR_ITEM_PREVIOUS = "Error: section [{0}] item has invalid value for 'previous' [{1}]" -ERR_ITEM_DICT_OR_LIST = "Error: section [{0}] value must be a dict or a list" -ERR_LIST_NOT_ALLOWED = "Error: section [{0}] item [{1}] has unexpected list value {2}" -ERR_BLCMD_MUST_BE_LIST = "Error: section [{0}] item [{1}] must be a list not [{2}]" -ERR_VALUE_CANNOT_BE_BOOLEAN = ( - "Error: section [{0}] item [{1}] value [{2}] must not " - "be a boolean - try quoting the value" -) - - -def get_supported_tuned_plugin_names(): - """get names of all tuned plugins supported by this module""" - return [ - "bootloader", - "modules", - "selinux", - "sysctl", - "sysfs", - "systemd", - "vm", - "cpu", - "disk", - "net", - "audio", - "scsi_host", - "eeepc_she", - "video", - "usb", - ] - - -def no_such_profile(): - """see if the last log message was that the profile did not exist""" - lastlogmsg = tuned.logs.root_logger.logstack[-1].msg - return re.search("Requested profile .* doesn't exist", lastlogmsg) is not None - - -def profile_to_dict(profile): - """convert profile object to dict""" - ret_val = {} - for unitname, unit in profile.units.items(): - ret_val[unitname] = unit.options - return ret_val - - -def debug_print_profile(profile, amodule): - """for debugging - print profile as INI""" - amodule.debug("profile {0}".format(profile.name)) - amodule.debug(str(profile_to_dict(profile))) - - -caught_name_error = None -try: - EMPTYUNIT = tuned.profiles.unit.Unit("empty", {}) -except NameError as nerr: - # tuned not loaded in check mode - caught_name_error = nerr - - -def get_profile_unit_key(profile, unitname, key): - """convenience function""" - return profile.units.get(unitname, EMPTYUNIT).options.get(key) - - -class BLCmdLine(object): - """A data type for handling bootloader cmdline values.""" - - def __init__(self, val): - self.key_list = [] # list of keys in order - to preserve ordering - self.key_to_val = {} # maps key to value - if val: - for item in shlex.split(val): - key, val = self.splititem(item) - self.key_list.append(key) - # None or '' means bare key - no value - self.key_to_val[key] = val - - @classmethod - def splititem(cls, item): - """split item in form key=somevalue into key and somevalue""" - # pylint: disable=blacklisted-name - key, _, val = item.partition("=") - return key, val - - @classmethod - def escapeval(cls, val): - """make sure val is quoted as in shell""" - return ansible_six_moves.shlex_quote(str(val)) - - def __str__(self): - vallist = [] - for key in self.key_list: - val = self.key_to_val[key] - if val: - vallist.append("{0}={1}".format(key, self.escapeval(val))) - else: - vallist.append(key) - return " ".join(vallist) - - def add(self, key, val): - """add/replace the given key & value""" - if key not in self.key_to_val: - self.key_list.append(key) - self.key_to_val[key] = val - - def remove(self, key): - """remove the given key""" - if key in self.key_to_val: - self.key_list.remove(key) - del self.key_to_val[key] - - -def apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue): - """apply a bootloader cmdline item to the current profile""" - name = item["name"] - if item.get(SECTION_TO_REPLACE, False): - blcmd = BLCmdLine("") - else: - blcmd = BLCmdLine(curvalue) - for subitem in item["value"]: - if "previous" in subitem: - continue - if subitem.get("state") == "absent": - blcmd.remove(subitem["name"]) - else: - blcmd.add(subitem["name"], subitem.get("value")) - blcmdstr = str(blcmd) - if blcmdstr: - current_profile.units.setdefault( - unitname, tuned.profiles.unit.Unit(unitname, {}) - ).options[name] = blcmdstr - elif curvalue: - del current_profile.units[unitname].options[name] - - -def apply_item_to_profile(item, unitname, current_profile): - """apply a specific item from a section to the current_profile""" - name = item["name"] - curvalue = get_profile_unit_key(current_profile, unitname, name) - newvalue = item.get("value") - if item.get("state", None) == "absent": - if curvalue: - del current_profile.units[unitname].options[name] - elif unitname == "bootloader" and name == "cmdline": - apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue) - else: - current_profile.units.setdefault( - unitname, tuned.profiles.unit.Unit(unitname, {}) - ).options[name] = str(newvalue) - - -def is_reboot_required(unitname): - """Some changes need a reboot for the changes to be applied - For example, bootloader cmdline changes""" - # for now, only bootloader cmdline changes need a reboot - return unitname == "bootloader" - - -def apply_params_to_profile(params, current_profile, purge): - """apply the settings from the input parameters to the current profile - delete the unit if it is empty after applying the parameter deletions - """ - changestatus = NOCHANGES - reboot_required = False - section_to_replace = params.pop(SECTION_TO_REPLACE, {}) - need_purge = set() - if purge: - # remove units not specified in params - need_purge = set(current_profile.units.keys()) - for unitname, items in params.items(): - unit = current_profile.units.get(unitname, None) - current_options = {} - if unit: - current_options = copy.deepcopy(unit.options) - replace = section_to_replace.get(unitname, purge) - if replace or (items == REMOVE_SECTION_VALUE): - if unit: - unit.options.clear() - if purge and unitname in need_purge: - need_purge.remove(unitname) - if items == REMOVE_SECTION_VALUE: - if unit: - # we changed the profile - changestatus = CHANGES - reboot_required = reboot_required or is_reboot_required(unitname) - # we're done - no further processing necessary for this unit - continue - for item in items: - if item and "previous" not in item: - apply_item_to_profile(item, unitname, current_profile) - newoptions = {} - if unitname in current_profile.units: - newoptions = current_profile.units[unitname].options - if current_options != newoptions: - changestatus = CHANGES - reboot_required = reboot_required or is_reboot_required(unitname) - for unitname in need_purge: - del current_profile.units[unitname] - changestatus = CHANGES - reboot_required = reboot_required or is_reboot_required(unitname) - # remove empty units - for unitname in list(current_profile.units.keys()): - if not current_profile.units[unitname].options: - del current_profile.units[unitname] - return changestatus, reboot_required - - -def write_profile(current_profile): - """write the profile to the profile file""" - # convert profile to configobj to write ini-style file - # profile.options go into [main] section - # profile.units go into [unitname] section - cfg = configobj.ConfigObj(indent_type="") - cfg.initial_comment = ["File managed by Ansible - DO NOT EDIT"] - cfg["main"] = current_profile.options - for unitname, unit in current_profile.units.items(): - cfg[unitname] = unit.options - profile_base_dir = tuned.consts.LOAD_DIRECTORIES[-1] - prof_fname = os.path.join( - profile_base_dir, TUNED_PROFILE, tuned.consts.PROFILE_FILE - ) - with open(prof_fname, "wb") as prof_f: - cfg.write(prof_f) - - -def update_current_profile_and_mode(daemon, profile_list): - """ensure that the tuned current_profile applies the kernel_settings last""" - changed = False - profile, manual = daemon._get_startup_profile() - # is TUNED_PROFILE in the list? - profile_list.extend(profile.split()) - if TUNED_PROFILE not in profile_list: - changed = True - profile_list.append(TUNED_PROFILE) - # have to convert to manual mode in order to ensure kernel_settings are applied - if not manual: - changed = True - manual = True - if changed: - daemon._save_active_profile(" ".join(profile_list), manual) - return changed - - -def setup_for_testing(): - """create an /etc/tuned and /usr/lib/tuned directory structure for testing""" - test_root_dir = os.environ.get("TEST_ROOT_DIR") - if test_root_dir is None: - test_root_dir = tempfile.mkdtemp(suffix=".lsr") - # copy all of the test configs and profiles - test_root_dir_tuned = os.path.join(test_root_dir, "tuned") - test_src_dir = os.environ.get("TEST_SRC_DIR", "tests") - src_dir = os.path.join(test_src_dir, "tuned") - shutil.copytree(src_dir, test_root_dir_tuned) - # patch all of the consts to use the test_root_dir - orig_consts = {} - for cnst in ( - "GLOBAL_CONFIG_FILE", - "ACTIVE_PROFILE_FILE", - "PROFILE_MODE_FILE", - "RECOMMEND_CONF_FILE", - "BOOT_CMDLINE_FILE", - ): - orig_consts[cnst] = tuned.consts.__dict__[cnst] - fname = os.path.join( - test_root_dir_tuned, - os.path.relpath(tuned.consts.__dict__[cnst], os.path.sep), - ) - tuned.consts.__dict__[cnst] = fname - dname = os.path.dirname(fname) - if not os.path.isdir(dname): - os.makedirs(dname) - orig_load_dirs = [] - for idx, dname in enumerate(tuned.consts.LOAD_DIRECTORIES): - orig_load_dirs.append(dname) - newdname = os.path.join( - test_root_dir_tuned, os.path.relpath(dname, os.path.sep) - ) - tuned.consts.LOAD_DIRECTORIES[idx] = newdname - if not os.path.isdir(newdname): - os.makedirs(newdname) - orig_rec_dirs = [] - for idx, dname in enumerate(tuned.consts.RECOMMEND_DIRECTORIES): - orig_rec_dirs.append(dname) - newdname = os.path.join( - test_root_dir_tuned, os.path.relpath(dname, os.path.sep) - ) - tuned.consts.RECOMMEND_DIRECTORIES[idx] = newdname - if not os.path.isdir(newdname): - os.makedirs(newdname) - has_func = bool( - getattr(tuned.utils.global_config.GlobalConfig.__init__, "__func__", False) - ) - if has_func: - orig_gc_init_defaults = ( - tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ - ) - orig_gc_load_config_defaults = ( - tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ - ) - if orig_gc_init_defaults: - tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - if orig_gc_load_config_defaults: - tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - else: - orig_gc_init_defaults = ( - tuned.utils.global_config.GlobalConfig.__init__.__defaults__ - ) - orig_gc_load_config_defaults = ( - tuned.utils.global_config.GlobalConfig.load_config.__defaults__ - ) - if orig_gc_init_defaults: - tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - if orig_gc_load_config_defaults: - tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - # this call fails on ubuntu and containers, so mock it for testing - import pyudev.monitor - - orig_set_receive_buffer_size = pyudev.monitor.Monitor.set_receive_buffer_size - pyudev.monitor.Monitor.set_receive_buffer_size = lambda self, size: None - - def test_cleanup(): - import os - import shutil - - if "TEST_ROOT_DIR" not in os.environ: - shutil.rmtree(test_root_dir) - for cnst, val in orig_consts.items(): - tuned.consts.__dict__[cnst] = val - for idx, dname in enumerate(orig_load_dirs): - tuned.consts.LOAD_DIRECTORIES[idx] = dname - for idx, dname in enumerate(orig_rec_dirs): - tuned.consts.RECOMMEND_DIRECTORIES[idx] = dname - if has_func: - tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( - orig_gc_init_defaults - ) - tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( - orig_gc_load_config_defaults - ) - else: - tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( - orig_gc_init_defaults - ) - tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( - orig_gc_load_config_defaults - ) - pyudev.monitor.Monitor.set_receive_buffer_size = orig_set_receive_buffer_size - - if "TEST_ROOT_DIR" not in os.environ: - atexit.register(test_cleanup) - return test_cleanup - - -def get_tuned_config(): - """get the tuned config and set our parameters in it""" - tuned_config = tuned.utils.global_config.GlobalConfig() - tuned_config.set("daemon", 0) - tuned_config.set("reapply_sysctl", 0) - tuned_config.set("dynamic_tuning", 0) - return tuned_config - - -def load_current_profile(tuned_config, tuned_profile, logger): - """load the current profile""" - tuned_app = None - errmsg = "Error loading tuned profile [{0}]".format(TUNED_PROFILE) - try: - tuned_app = tuned.daemon.Application(tuned_profile, tuned_config) - except TunedException as tex: - logger.debug("caught TunedException [{0}]".format(tex)) - errmsg = errmsg + ": {0}".format(tex) - tuned_app = None - except IOError as ioe: - # for testing this case, need to create a profile with a bad permissions e.g. - # mkdir ioerror_profile; touch ioerror_profile/tuned.conf; chmod 0000 !$ - logger.debug("caught IOError [{0}]".format(ioe)) - errmsg = errmsg + ": {0}".format(ioe) - tuned_app = None - if ( - tuned_app is None - or tuned_app.daemon is None - or tuned_app.daemon.profile is None - or tuned_app.daemon.profile.units is None - or tuned_app.daemon.profile.options is None - or "summary" not in tuned_app.daemon.profile.options - ): - tuned_app = None - if no_such_profile(): - errmsg = errmsg + ": Profile does not exist" - return tuned_app, errmsg - - -def validate_and_digest_item(sectionname, item, listallowed=True, allowempty=False): - """Validate an item - must contain only name, value, and state""" - tmpitem = item.copy() - name = tmpitem.pop("name", None) - value = tmpitem.pop("value", None) - state = tmpitem.pop("state", None) - errlist = [] - if name is None: - errlist.append(ERR_SECTION_MISSING_NAME.format(sectionname, item)) - elif not isinstance(name, ansible_six.string_types): - errlist.append(ERR_NAME_NOT_VALID.format(sectionname, name)) - elif (value is None and not allowempty) and state is None: - errlist.append(ERR_NO_VALUE_OR_STATE.format(sectionname, name)) - elif value is not None and state is not None: - errlist.append(ERR_BOTH_VALUE_AND_STATE.format(sectionname, name)) - elif state is not None and state != "absent": - errlist.append(ERR_STATE_ABSENT.format(sectionname, name, state)) - elif tmpitem: - errlist.append(ERR_UNEXPECTED_VALUES.format(sectionname, name, tmpitem)) - elif isinstance(value, list): - if not listallowed: - errlist.append(ERR_LIST_NOT_ALLOWED.format(sectionname, name, value)) - else: - for valitem in value: - if not isinstance(valitem, dict): - errlist.append( - ERR_VALUE_ITEM_NOT_DICT.format(sectionname, name, valitem) - ) - elif "previous" in valitem: - if valitem["previous"] != "replaced": - errlist.append( - ERR_VALUE_ITEM_PREVIOUS.format( - sectionname, name, valitem["previous"] - ) - ) - else: - item[SECTION_TO_REPLACE] = True - else: - tmperrlist = validate_and_digest_item( - sectionname, valitem, False, True - ) - errlist.extend(tmperrlist) - elif sectionname == "bootloader" and name == "cmdline": - errlist.append(ERR_BLCMD_MUST_BE_LIST.format(sectionname, name, value)) - elif isinstance(value, bool): - errlist.append(ERR_VALUE_CANNOT_BE_BOOLEAN.format(sectionname, name, value)) - return errlist - - -def validate_and_digest(params): - """Validate that params is in the correct format, since we - are using type `raw`, we have to perform the validation here. - Also do some pre-processing to make it easier to apply - the params to profile""" - errlist = [] - replaces = {} - for sectionname, items in params.items(): - if isinstance(items, dict): - if not items == REMOVE_SECTION_VALUE: - errlist.append( - ERR_REMOVE_SECTION_VALUE.format(sectionname, REMOVE_SECTION_VALUE) - ) - elif isinstance(items, list): - for item in items: - if not isinstance(item, dict): - errlist.append(ERR_ITEM_NOT_DICT.format(sectionname, item)) - elif not item: - continue # ignore empty items - elif "previous" in item: - if item["previous"] != "replaced": - errlist.append( - ERR_ITEM_PREVIOUS.format(sectionname, item["previous"]) - ) - else: - replaces[sectionname] = True - else: - itemerrlist = validate_and_digest_item(sectionname, item) - errlist.extend(itemerrlist) - else: - errlist.append(ERR_ITEM_DICT_OR_LIST.format(sectionname)) - if replaces: - params[SECTION_TO_REPLACE] = replaces - - return errlist - - -def remove_if_empty(params): - """recursively remove empty items from params - return true if params results in being empty, - false otherwise""" - if isinstance(params, list): - removed = 0 - for idx in range(0, len(params)): - realidx = idx - removed - if remove_if_empty(params[realidx]): - del params[realidx] - removed = removed + 1 - elif isinstance(params, dict): - for key, val in list(params.items()): - if remove_if_empty(val): - del params[key] - return params == [] or params == {} or params == "" or params is None - + with open(filename) as f: + return f.read().rstrip() + except FileNotFoundError as e: + print('safe_file_get_contents: suppressed exception FileNotFoundError with content \'%s\'' % e) + +def get_sysfs_fields(): + result = {} + result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] + result["kernel_settings_transparent_hugepages_defrag"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0] + + # will collect a list of cpus associated to the template machine, but only use settings from the first one + cpus = pyudev.Context().list_devices(subsystem="cpu") + num_cpus = 0 + + for cpu in cpus: + num_cpus += 1 + if 'kernel_settings_cpu_governor' not in result: + result["kernel_settings_cpu_governor"] = safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path) + result["kernel_settings_sampling_down_factor"] = safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % result['kernel_settings_cpu_governor']) + + print("get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus) + + result["kernel_settings_cpu_min_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct") + result["kernel_settings_cpu_max_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct") + result["kernel_settings_cpu_no_turbo"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo") + + # will collect a list of blocks associated to the template machine, but only use settings from the first one + blocks = pyudev.Context().list_devices(subsystem="block") + num_blocks = 0 + + for block in blocks: + num_blocks += 1 + if 'kernel_settings_disk_elevator' not in result: + result["kernel_settings_disk_elevator"] = re.findall(r'\[(\w+)\]', safe_file_get_contents("%s/queue/scheduler" % block.sys_path)) + result["kernel_settings_disk_read_ahead_kb"] = int(safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)) + result["kernel_settings_disk_scheduler_quantum"] = safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path) + + print("get_sysfs_fields: found %d blocks associated to the template machine" % num_blocks) + + result["kernel_settings_avc_cache_threshold"] = int(safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")) + result["kernel_settings_nf_conntrack_hashsize"] = int(safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize")) + + # will collect a list of modules associated to the template machine, but only use settings from the first one + num_modules = 0 + devices = pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") + + for device in devices: + module_name = device.parent.driver + if module_name in ["snd_hda_intel", "snd_ac97_codec"]: + num_modules += 1 + result["kernel_settings_audio_timeout"] = safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name) + result["kernel_settings_audio_reset_controller"] = safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name) + + print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_modules) + + # will collect a list of scsis associated to the template machine, but only use settings from the first one + num_scsis = 0 + scsis = pyudev.Context().list_devices(subsystem="scsi") + + for scsi in scsis: + num_scsis += 1 + if "kernel_settings_scsi_host_alpm" not in result: + result["kernel_settings_scsi_host_alpm"] = safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path) + + print("get_sysfs_fields: found %d scsis associated to the template machine" % num_scsis) + + # will collect a list of graphics cards associated to the template machine, but only use settings from the first one + num_gcards = 0 + gcards = pyudev.Context().list_devices(subsystem="drm").match_sys_name("card*").match_property("DEVTYPE", "drm_minor") + + for gcard in gcards: + num_gcards += 1 + method = safe_file_get_contents("%s/device/power_method" % gcard.sys_path) + if "kernel_settings_video_radeon_powersave" not in result: + if method == "profile": + result["kernel_settings_video_radeon_powersave"] = safe_file_get_contents("%s/device/power_profile" % gcard.sys_path) + elif method == "dynpm": + result["kernel_settings_video_radeon_powersave"] = "dynpm" + elif method == "dpm": + result["kernel_settings_video_radeon_powersave"] = "dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path) + + print("get_sysfs_fields: found %d gcards associated to the template machine" % num_gcards) + + # will collect a list of usb interfaces associated to the template machine, but only use settings from the first one + num_usbs = 0 + usbs = pyudev.Context().list_devices(subsystem="usb") + + for usb in usbs: + num_usbs += 1 + if "kernel_settings_usb_autosuspend" not in result: + result["kernel_settings_usb_autosuspend"] = safe_file_get_contents("%s/power/autosuspend" % usb.sys_path) + + print("get_sysfs_fields: found %d usbs associated to the template machine" % num_usbs) + + return result def run_module(): - """ The entry point of the module. """ + module_args = dict() - module_args = dict( - purge=dict(type="bool", required=False, default=False), + result = dict( + changed=False, + ansible_facts=dict(), ) - tuned_plugin_names = get_supported_tuned_plugin_names() - for plugin_name in tuned_plugin_names: - # use raw here - type can be dict or list - perform validation - # below - module_args[plugin_name] = dict(type="raw", required=False) - - result = dict(changed=False, message="") - module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) - - if not module.check_mode: - if os.environ.get("TESTING", "false") == "true": - # pylint: disable=blacklisted-name - _ = setup_for_testing() - - params = module.params - # remove any non-tuned fields from params and save them locally - # state = params.pop("state") - purge = params.pop("purge", False) - # also remove any empty or None - # pylint: disable=blacklisted-name - _ = remove_if_empty(params) - errlist = validate_and_digest(params) - if errlist: - errmsg = "Invalid format for input parameters" - module.fail_json(msg=errmsg, warnings=errlist, **result) + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) - # In check_mode, just perform input validation (above), because - # tuned will not be installed on the remote system if module.check_mode: module.exit_json(**result) - elif caught_import_error is not None: - raise caught_import_error # pylint: disable-msg=E0702 - elif caught_name_error is not None: - # name error is usually because tuned module was not imported - # but just in case, report it here - raise caught_name_error # pylint: disable-msg=E0702 - tuned_config = get_tuned_config() - current_profile = None - tuned_app, errmsg = load_current_profile(tuned_config, TUNED_PROFILE, module) + result['ansible_facts'] = get_sysfs_fields() - if tuned_app is None: - module.fail_json(msg=errmsg, **result) - else: - current_profile = tuned_app.daemon.profile - debug_print_profile(current_profile, module) - errmsg = "" - - result["msg"] = "Kernel settings were updated." - - # apply the given params to the profile - if there are any new items - # the function will return True we set changed = True - changestatus, reboot_required = apply_params_to_profile( - params, current_profile, purge - ) - profile_list = [] - if update_current_profile_and_mode(tuned_app.daemon, profile_list): - # profile or mode changed - if changestatus == NOCHANGES: - changestatus = CHANGES - result["msg"] = "Updated active profile and/or mode." - if changestatus > NOCHANGES: - try: - write_profile(current_profile) - # notify tuned to reload/reapply profile - except TunedException as tex: - module.debug("caught TunedException [{0}]".format(tex)) - errmsg = "Unable to apply tuned settings: {0}".format(tex) - module.fail_json(msg=errmsg, **result) - except IOError as ioe: - module.debug("caught IOError [{0}]".format(ioe)) - errmsg = "Unable to apply tuned settings: {0}".format(ioe) - module.fail_json(msg=errmsg, **result) - result["changed"] = True - else: - result["msg"] = "Kernel settings are up to date." - debug_print_profile(current_profile, module) - result["new_profile"] = profile_to_dict(current_profile) - result["active_profile"] = " ".join(profile_list) - result["reboot_required"] = reboot_required - if reboot_required: - result["msg"] = ( - result["msg"] + " A system reboot is needed to apply the changes." - ) module.exit_json(**result) - def main(): - """ The main function! """ run_module() - -if __name__ == "__main__": - main() +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/library/old_kernel_settings.py b/library/old_kernel_settings.py new file mode 100644 index 00000000..4aa7b02a --- /dev/null +++ b/library/old_kernel_settings.py @@ -0,0 +1,892 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2020, Rich Megginson +# SPDX-License-Identifier: GPL-2.0-or-later +# +""" Manage kernel settings using tuned via a wrapper """ + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + +DOCUMENTATION = """ +--- +module: kernel_settings + +short_description: Manage kernel settings using tuned via a wrapper + +version_added: "2.8" + +description: + - | + Manage kernel settings using tuned via a wrapper. The options correspond + to names of units or plugins in tuned. For example, the option C(sysctl) + corresponds to the C(sysctl) unit or plugin in tuned. Setting parameters + works mostly like it does with tuned, except that this module uses Ansible + YAML format instead of the tuned INI-style profile file format. This module + creates a special tuned profile C(kernel_settings) which will be applied by + tuned before any other profiles, allowing the user to configure tuned to + override settings made by this module. You should be aware of this if you + plan to use tuned in addition to using this module. + - HORIZONTALLINE + - | + NOTE: the options list may be incomplete - the actual options are generated + dynamically from tuned, for the current options supported by the version + of tuned, which are the tuned supported plugins. Only the most common + options are listed. See the tuned documentation for the full list and + more information. + - HORIZONTALLINE + - | + Each option takes a list or dict of settings. Each setting is a C(dict). + The C(dict) must have one of the keys C(name), C(value), C(state), or C(previous). + C(state) is used to remove settings or sections of settings. C(previous) is + used to replace all of values in a section with the given values. The only case + where an option takes a dict is when you want to remove a section completely - + then value for the section is the dict C({"state":"empty"}). + If you specify multiple settings with the same name in a section, the last one + will be used. +options: + sysctl: + description: + - | + list of sysctl settings to apply - this works mostly + like the C(sysctl) module except that C(/etc/sysctl.conf) and files + under C(/etc/sysctl.d) are not used. + required: false + sysfs: + description: + - key/value pairs of sysfs settings to apply + required: false + bootloader: + description: + - the C(cmdline) option can be used to set, add, or delete + kernel command line options. See EXAMPLES for some examples + of how to use this option. + Note that this uses the tuned implementation, which adds these + options to whatever the default bootloader command line arguments + tuned is historically used to add/delete performance related + kernel command line arguments e.g. C(spectre_v2=off). If you need + more general purpose bootloader configuration, you should use + a bootloader module/role. + type: raw + purge: + description: + - Remove the current kernel_settings, whatever they are, and force + the given settings to be the current and only settings + type: bool + default: false + +author: + - Rich Megginson (@richm) +""" + +EXAMPLES = """ +# Add or replace the sysctl `fs.file-max` parameter with the value 65535. The +# existing settings are not touched. +- name: Add or replace the sysctl `fs.file-max` parameter with the value 65535. + kernel_settings: + sysctl: + - name: fs.file-max + value: 65535 + +- name: remove the entire sysctl section + kernel_settings: + sysctl: + state: empty + +- name: remove the entire sysctl section and replace with the given values + kernel_settings: + sysctl: + - previous: replaced + - name: fs.file-max + value: 65535 + +- name: add the sysctl vm.max_mmap_regions, and disable spectre/meltdown security + kernel_settings: + sysctl: + - name: vm.max_mmap_regions + value: 262144 + sysfs: + - name: /sys/kernel/debug/x86/pti_enabled + value: 0 + - name: /sys/kernel/debug/x86/retp_enabled + value: 0 + - name: /sys/kernel/debug/x86/ibrs_enabled + value: 0 + +# replace the existing sysctl section with the specified section +# delete the /sys/kernel/debug/x86/retp_enabled setting +# completely remove the vm section +# add the bootloader cmdline arguments spectre_v2=off nopti +# remove the bootloader cmdline arguments panic splash +- name: more settings + kernel_settings: + sysctl: + - previous: replaced + - name: vm.max_mmap_regions + value: 262144 + sysfs: + - name: /sys/kernel/debug/x86/retp_enabled + state: absent + vm: + state: empty + bootloader: + - name: cmdline + value: + - name: spectre_v2 + value: off + - name: nopti + - name: panic + state: absent + - name: splash + state: absent +""" + +RETURN = """ +msg: + description: | + A short text message to say what action this module performed. + returned: always + type: str +new_profile: + description: | + This is the tuned profile in dict format, after the changes, if any, + have been applied. + returned: always + type: dict +reboot_required: + description: | + default C(false) - if true, this means a reboot of the managed host is + required in order for the changes to take effect. + returned: always + type: bool +active_profile: + description: | + This is the space delimited list of active profiles as reported + by tuned. + returned: always + type: str +""" + +import os +import logging +import re +import tempfile +import shutil +import shlex +import contextlib +import atexit # for testing +import copy + +try: + import configobj + + HAS_CONFIGOBJ = True +except ImportError: + HAS_CONFIGOBJ = False +import ansible.module_utils.six as ansible_six + +# This is a bit of a mystery - bug in pylint? +# pylint: disable=import-error +import ansible.module_utils.six.moves as ansible_six_moves + +from ansible.module_utils.basic import AnsibleModule + +# see https://github.com/python/cpython/blob/master/Lib/logging/__init__.py +# for information about logging module internals + + +class TunedLogWrapper(logging.getLoggerClass()): + """This wraps the tuned logger so that we can intercept logs and handle them here""" + + def __init__(self, *args, **kwargs): + super(TunedLogWrapper, self).__init__(*args, **kwargs) + self.setLevel(logging.DEBUG) + self.logstack = [] + + def handle(self, record): + self.logstack.append(record) + + +@contextlib.contextmanager +def save_and_restore_logging_methods(): + """do not allow tuned logging to pollute global logging module or + ansible logging""" + save_logging_add_level_name = logging.addLevelName + + def wrapper_add_level_name(_levelval, _levelname): + """ignore tuned.logging call to logging.addLevelName""" + # print('addLevelName wrapper ignoring {} {}'.format(levelval, levelname)) + + logging.addLevelName = wrapper_add_level_name + save_logging_set_logger_class = logging.setLoggerClass + + def wrapper_set_logger_class(_clsname): + """ignore tuned.logging call to logging.setLoggerClass""" + # print('setLoggerClass wrapper ignoring {}'.format(clsname)) + + logging.setLoggerClass = wrapper_set_logger_class + try: + yield + finally: + logging.addLevelName = save_logging_add_level_name + logging.setLoggerClass = save_logging_set_logger_class + + +caught_import_error = None +HAS_TUNED = False +try: + with save_and_restore_logging_methods(): + import tuned.logs + HAS_TUNED = True +except ImportError as ierr: + # tuned package might not be available in check mode - so just + # note that this is missing, and do not report in check mode + caught_import_error = ierr + +if HAS_TUNED: + tuned.logs.root_logger = TunedLogWrapper(__name__) + tuned.logs.get = lambda: tuned.logs.root_logger + + import tuned.consts + import tuned.utils.global_config + import tuned.daemon + from tuned.exceptions import TunedException + + +TUNED_PROFILE = os.environ.get("TEST_PROFILE", "kernel_settings") +NOCHANGES = 0 +CHANGES = 1 +REMOVE_SECTION_VALUE = {"state": "empty"} +SECTION_TO_REPLACE = "__section_to_replace" +ERR_SECTION_MISSING_NAME = "Error: section [{0}] item is missing 'name': {1}" +ERR_NAME_NOT_VALID = "Error: section [{0}] item name [{1}] is not a valid string" +ERR_NO_VALUE_OR_STATE = ( + "Error: section [{0}] item name [{1}] must have either a 'value' or 'state'" +) +ERR_BOTH_VALUE_AND_STATE = ( + "Error: section [{0}] item name [{1}] must have only one of 'value' or 'state'" +) +ERR_STATE_ABSENT = ( + "Error: section [{0}] item name [{1}] state value must be 'absent' not [{2}]" +) +ERR_UNEXPECTED_VALUES = "Error: section [{0}] item [{1}] has unexpected values {2}" +ERR_VALUE_ITEM_NOT_DICT = ( + "Error: section [{0}] item name [{1}] value [{2}] is not a dict" +) +ERR_VALUE_ITEM_PREVIOUS = ( + "Error: section [{0}] item name [{1}] has invalid value for 'previous' [{2}]" +) +ERR_REMOVE_SECTION_VALUE = "Error: to remove the section [{0}] specify the value {1}" +ERR_ITEM_NOT_DICT = "Error: section [{0}] item value [{1}] is not a dict" +ERR_ITEM_PREVIOUS = "Error: section [{0}] item has invalid value for 'previous' [{1}]" +ERR_ITEM_DICT_OR_LIST = "Error: section [{0}] value must be a dict or a list" +ERR_LIST_NOT_ALLOWED = "Error: section [{0}] item [{1}] has unexpected list value {2}" +ERR_BLCMD_MUST_BE_LIST = "Error: section [{0}] item [{1}] must be a list not [{2}]" +ERR_VALUE_CANNOT_BE_BOOLEAN = ( + "Error: section [{0}] item [{1}] value [{2}] must not " + "be a boolean - try quoting the value" +) + + +def get_supported_tuned_plugin_names(): + """get names of all tuned plugins supported by this module""" + return [ + "bootloader", + "modules", + "selinux", + "sysctl", + "sysfs", + "systemd", + "vm", + "cpu", + "disk", + "net", + "audio", + "scsi_host", + "video", + "usb", + ] + + +def no_such_profile(): + """see if the last log message was that the profile did not exist""" + lastlogmsg = tuned.logs.root_logger.logstack[-1].msg + return re.search("Requested profile .* doesn't exist", lastlogmsg) is not None + + +def profile_to_dict(profile): + """convert profile object to dict""" + ret_val = {} + for unitname, unit in profile.units.items(): + ret_val[unitname] = unit.options + return ret_val + + +def debug_print_profile(profile, amodule): + """for debugging - print profile as INI""" + amodule.debug("profile {0}".format(profile.name)) + amodule.debug(str(profile_to_dict(profile))) + + +caught_name_error = None +try: + EMPTYUNIT = tuned.profiles.unit.Unit("empty", {}) +except NameError as nerr: + # tuned not loaded in check mode + caught_name_error = nerr + + +def get_profile_unit_key(profile, unitname, key): + """convenience function""" + return profile.units.get(unitname, EMPTYUNIT).options.get(key) + + +class BLCmdLine(object): + """A data type for handling bootloader cmdline values.""" + + def __init__(self, val): + self.key_list = [] # list of keys in order - to preserve ordering + self.key_to_val = {} # maps key to value + if val: + for item in shlex.split(val): + key, val = self.splititem(item) + self.key_list.append(key) + # None or '' means bare key - no value + self.key_to_val[key] = val + + @classmethod + def splititem(cls, item): + """split item in form key=somevalue into key and somevalue""" + # pylint: disable=blacklisted-name + key, _, val = item.partition("=") + return key, val + + @classmethod + def escapeval(cls, val): + """make sure val is quoted as in shell""" + return ansible_six_moves.shlex_quote(str(val)) + + def __str__(self): + vallist = [] + for key in self.key_list: + val = self.key_to_val[key] + if val: + vallist.append("{0}={1}".format(key, self.escapeval(val))) + else: + vallist.append(key) + return " ".join(vallist) + + def add(self, key, val): + """add/replace the given key & value""" + if key not in self.key_to_val: + self.key_list.append(key) + self.key_to_val[key] = val + + def remove(self, key): + """remove the given key""" + if key in self.key_to_val: + self.key_list.remove(key) + del self.key_to_val[key] + + +def apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue): + """apply a bootloader cmdline item to the current profile""" + name = item["name"] + if item.get(SECTION_TO_REPLACE, False): + blcmd = BLCmdLine("") + else: + blcmd = BLCmdLine(curvalue) + for subitem in item["value"]: + if "previous" in subitem: + continue + if subitem.get("state") == "absent": + blcmd.remove(subitem["name"]) + else: + blcmd.add(subitem["name"], subitem.get("value")) + blcmdstr = str(blcmd) + if blcmdstr: + current_profile.units.setdefault( + unitname, tuned.profiles.unit.Unit(unitname, {}) + ).options[name] = blcmdstr + elif curvalue: + del current_profile.units[unitname].options[name] + + +def apply_item_to_profile(item, unitname, current_profile): + """apply a specific item from a section to the current_profile""" + name = item["name"] + curvalue = get_profile_unit_key(current_profile, unitname, name) + newvalue = item.get("value") + if item.get("state", None) == "absent": + if curvalue: + del current_profile.units[unitname].options[name] + elif unitname == "bootloader" and name == "cmdline": + apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue) + else: + current_profile.units.setdefault( + unitname, tuned.profiles.unit.Unit(unitname, {}) + ).options[name] = str(newvalue) + + +def is_reboot_required(unitname): + """Some changes need a reboot for the changes to be applied + For example, bootloader cmdline changes""" + # for now, only bootloader cmdline changes need a reboot + return unitname == "bootloader" + + +def apply_params_to_profile(params, current_profile, purge): + """apply the settings from the input parameters to the current profile + delete the unit if it is empty after applying the parameter deletions + """ + changestatus = NOCHANGES + reboot_required = False + section_to_replace = params.pop(SECTION_TO_REPLACE, {}) + need_purge = set() + if purge: + # remove units not specified in params + need_purge = set(current_profile.units.keys()) + for unitname, items in params.items(): + unit = current_profile.units.get(unitname, None) + current_options = {} + if unit: + current_options = copy.deepcopy(unit.options) + replace = section_to_replace.get(unitname, purge) + if replace or (items == REMOVE_SECTION_VALUE): + if unit: + unit.options.clear() + if purge and unitname in need_purge: + need_purge.remove(unitname) + if items == REMOVE_SECTION_VALUE: + if unit: + # we changed the profile + changestatus = CHANGES + reboot_required = reboot_required or is_reboot_required(unitname) + # we're done - no further processing necessary for this unit + continue + for item in items: + if item and "previous" not in item: + apply_item_to_profile(item, unitname, current_profile) + newoptions = {} + if unitname in current_profile.units: + newoptions = current_profile.units[unitname].options + if current_options != newoptions: + changestatus = CHANGES + reboot_required = reboot_required or is_reboot_required(unitname) + for unitname in need_purge: + del current_profile.units[unitname] + changestatus = CHANGES + reboot_required = reboot_required or is_reboot_required(unitname) + # remove empty units + for unitname in list(current_profile.units.keys()): + if not current_profile.units[unitname].options: + del current_profile.units[unitname] + return changestatus, reboot_required + + +def write_profile(current_profile): + """write the profile to the profile file""" + # convert profile to configobj to write ini-style file + # profile.options go into [main] section + # profile.units go into [unitname] section + cfg = configobj.ConfigObj(indent_type="") + cfg.initial_comment = ["File managed by Ansible - DO NOT EDIT"] + cfg["main"] = current_profile.options + for unitname, unit in current_profile.units.items(): + cfg[unitname] = unit.options + profile_base_dir = tuned.consts.LOAD_DIRECTORIES[-1] + prof_fname = os.path.join( + profile_base_dir, TUNED_PROFILE, tuned.consts.PROFILE_FILE + ) + with open(prof_fname, "wb") as prof_f: + cfg.write(prof_f) + + +def update_current_profile_and_mode(daemon, profile_list): + """ensure that the tuned current_profile applies the kernel_settings last""" + changed = False + profile, manual = daemon._get_startup_profile() + # is TUNED_PROFILE in the list? + profile_list.extend(profile.split()) + if TUNED_PROFILE not in profile_list: + changed = True + profile_list.append(TUNED_PROFILE) + # have to convert to manual mode in order to ensure kernel_settings are applied + if not manual: + changed = True + manual = True + if changed: + daemon._save_active_profile(" ".join(profile_list), manual) + return changed + + +def setup_for_testing(): + """create an /etc/tuned and /usr/lib/tuned directory structure for testing""" + test_root_dir = os.environ.get("TEST_ROOT_DIR") + if test_root_dir is None: + test_root_dir = tempfile.mkdtemp(suffix=".lsr") + # copy all of the test configs and profiles + test_root_dir_tuned = os.path.join(test_root_dir, "tuned") + test_src_dir = os.environ.get("TEST_SRC_DIR", "tests") + src_dir = os.path.join(test_src_dir, "tuned") + shutil.copytree(src_dir, test_root_dir_tuned) + # patch all of the consts to use the test_root_dir + orig_consts = {} + for cnst in ( + "GLOBAL_CONFIG_FILE", + "ACTIVE_PROFILE_FILE", + "PROFILE_MODE_FILE", + "RECOMMEND_CONF_FILE", + "BOOT_CMDLINE_FILE", + ): + orig_consts[cnst] = tuned.consts.__dict__[cnst] + fname = os.path.join( + test_root_dir_tuned, + os.path.relpath(tuned.consts.__dict__[cnst], os.path.sep), + ) + tuned.consts.__dict__[cnst] = fname + dname = os.path.dirname(fname) + if not os.path.isdir(dname): + os.makedirs(dname) + orig_load_dirs = [] + for idx, dname in enumerate(tuned.consts.LOAD_DIRECTORIES): + orig_load_dirs.append(dname) + newdname = os.path.join( + test_root_dir_tuned, os.path.relpath(dname, os.path.sep) + ) + tuned.consts.LOAD_DIRECTORIES[idx] = newdname + if not os.path.isdir(newdname): + os.makedirs(newdname) + orig_rec_dirs = [] + for idx, dname in enumerate(tuned.consts.RECOMMEND_DIRECTORIES): + orig_rec_dirs.append(dname) + newdname = os.path.join( + test_root_dir_tuned, os.path.relpath(dname, os.path.sep) + ) + tuned.consts.RECOMMEND_DIRECTORIES[idx] = newdname + if not os.path.isdir(newdname): + os.makedirs(newdname) + has_func = bool( + getattr(tuned.utils.global_config.GlobalConfig.__init__, "__func__", False) + ) + if has_func: + orig_gc_init_defaults = ( + tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ + ) + orig_gc_load_config_defaults = ( + tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ + ) + if orig_gc_init_defaults: + tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + if orig_gc_load_config_defaults: + tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + else: + orig_gc_init_defaults = ( + tuned.utils.global_config.GlobalConfig.__init__.__defaults__ + ) + orig_gc_load_config_defaults = ( + tuned.utils.global_config.GlobalConfig.load_config.__defaults__ + ) + if orig_gc_init_defaults: + tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + if orig_gc_load_config_defaults: + tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + # this call fails on ubuntu and containers, so mock it for testing + import pyudev.monitor + + orig_set_receive_buffer_size = pyudev.monitor.Monitor.set_receive_buffer_size + pyudev.monitor.Monitor.set_receive_buffer_size = lambda self, size: None + + def test_cleanup(): + import os + import shutil + + if "TEST_ROOT_DIR" not in os.environ: + shutil.rmtree(test_root_dir) + for cnst, val in orig_consts.items(): + tuned.consts.__dict__[cnst] = val + for idx, dname in enumerate(orig_load_dirs): + tuned.consts.LOAD_DIRECTORIES[idx] = dname + for idx, dname in enumerate(orig_rec_dirs): + tuned.consts.RECOMMEND_DIRECTORIES[idx] = dname + if has_func: + tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( + orig_gc_init_defaults + ) + tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( + orig_gc_load_config_defaults + ) + else: + tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( + orig_gc_init_defaults + ) + tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( + orig_gc_load_config_defaults + ) + pyudev.monitor.Monitor.set_receive_buffer_size = orig_set_receive_buffer_size + + if "TEST_ROOT_DIR" not in os.environ: + atexit.register(test_cleanup) + return test_cleanup + + +def get_tuned_config(): + """get the tuned config and set our parameters in it""" + tuned_config = tuned.utils.global_config.GlobalConfig() + tuned_config.set("daemon", 0) + tuned_config.set("reapply_sysctl", 0) + tuned_config.set("dynamic_tuning", 0) + return tuned_config + + +def load_current_profile(tuned_config, tuned_profile, logger): + """load the current profile""" + tuned_app = None + errmsg = "Error loading tuned profile [{0}]".format(TUNED_PROFILE) + try: + tuned_app = tuned.daemon.Application(tuned_profile, tuned_config) + except TunedException as tex: + logger.debug("caught TunedException [{0}]".format(tex)) + errmsg = errmsg + ": {0}".format(tex) + tuned_app = None + except IOError as ioe: + # for testing this case, need to create a profile with a bad permissions e.g. + # mkdir ioerror_profile; touch ioerror_profile/tuned.conf; chmod 0000 !$ + logger.debug("caught IOError [{0}]".format(ioe)) + errmsg = errmsg + ": {0}".format(ioe) + tuned_app = None + if ( + tuned_app is None + or tuned_app.daemon is None + or tuned_app.daemon.profile is None + or tuned_app.daemon.profile.units is None + or tuned_app.daemon.profile.options is None + or "summary" not in tuned_app.daemon.profile.options + ): + tuned_app = None + if no_such_profile(): + errmsg = errmsg + ": Profile does not exist" + return tuned_app, errmsg + + +def validate_and_digest_item(sectionname, item, listallowed=True, allowempty=False): + """Validate an item - must contain only name, value, and state""" + tmpitem = item.copy() + name = tmpitem.pop("name", None) + value = tmpitem.pop("value", None) + state = tmpitem.pop("state", None) + errlist = [] + if name is None: + errlist.append(ERR_SECTION_MISSING_NAME.format(sectionname, item)) + elif not isinstance(name, ansible_six.string_types): + errlist.append(ERR_NAME_NOT_VALID.format(sectionname, name)) + elif (value is None and not allowempty) and state is None: + errlist.append(ERR_NO_VALUE_OR_STATE.format(sectionname, name)) + elif value is not None and state is not None: + errlist.append(ERR_BOTH_VALUE_AND_STATE.format(sectionname, name)) + elif state is not None and state != "absent": + errlist.append(ERR_STATE_ABSENT.format(sectionname, name, state)) + elif tmpitem: + errlist.append(ERR_UNEXPECTED_VALUES.format(sectionname, name, tmpitem)) + elif isinstance(value, list): + if not listallowed: + errlist.append(ERR_LIST_NOT_ALLOWED.format(sectionname, name, value)) + else: + for valitem in value: + if not isinstance(valitem, dict): + errlist.append( + ERR_VALUE_ITEM_NOT_DICT.format(sectionname, name, valitem) + ) + elif "previous" in valitem: + if valitem["previous"] != "replaced": + errlist.append( + ERR_VALUE_ITEM_PREVIOUS.format( + sectionname, name, valitem["previous"] + ) + ) + else: + item[SECTION_TO_REPLACE] = True + else: + tmperrlist = validate_and_digest_item( + sectionname, valitem, False, True + ) + errlist.extend(tmperrlist) + elif sectionname == "bootloader" and name == "cmdline": + errlist.append(ERR_BLCMD_MUST_BE_LIST.format(sectionname, name, value)) + elif isinstance(value, bool): + errlist.append(ERR_VALUE_CANNOT_BE_BOOLEAN.format(sectionname, name, value)) + return errlist + + +def validate_and_digest(params): + """Validate that params is in the correct format, since we + are using type `raw`, we have to perform the validation here. + Also do some pre-processing to make it easier to apply + the params to profile""" + errlist = [] + replaces = {} + for sectionname, items in params.items(): + if isinstance(items, dict): + if not items == REMOVE_SECTION_VALUE: + errlist.append( + ERR_REMOVE_SECTION_VALUE.format(sectionname, REMOVE_SECTION_VALUE) + ) + elif isinstance(items, list): + for item in items: + if not isinstance(item, dict): + errlist.append(ERR_ITEM_NOT_DICT.format(sectionname, item)) + elif not item: + continue # ignore empty items + elif "previous" in item: + if item["previous"] != "replaced": + errlist.append( + ERR_ITEM_PREVIOUS.format(sectionname, item["previous"]) + ) + else: + replaces[sectionname] = True + else: + itemerrlist = validate_and_digest_item(sectionname, item) + errlist.extend(itemerrlist) + else: + errlist.append(ERR_ITEM_DICT_OR_LIST.format(sectionname)) + if replaces: + params[SECTION_TO_REPLACE] = replaces + + return errlist + + +def remove_if_empty(params): + """recursively remove empty items from params + return true if params results in being empty, + false otherwise""" + if isinstance(params, list): + removed = 0 + for idx in range(0, len(params)): + realidx = idx - removed + if remove_if_empty(params[realidx]): + del params[realidx] + removed = removed + 1 + elif isinstance(params, dict): + for key, val in list(params.items()): + if remove_if_empty(val): + del params[key] + return params == [] or params == {} or params == "" or params is None + + +def run_module(): + """ The entry point of the module. """ + + module_args = dict( + purge=dict(type="bool", required=False, default=False), + ) + tuned_plugin_names = get_supported_tuned_plugin_names() + for plugin_name in tuned_plugin_names: + # use raw here - type can be dict or list - perform validation + # below + module_args[plugin_name] = dict(type="raw", required=False) + + result = dict(changed=False, message="") + + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) + + if not module.check_mode: + if os.environ.get("TESTING", "false") == "true": + # pylint: disable=blacklisted-name + _ = setup_for_testing() + + params = module.params + # remove any non-tuned fields from params and save them locally + # state = params.pop("state") + purge = params.pop("purge", False) + # also remove any empty or None + # pylint: disable=blacklisted-name + _ = remove_if_empty(params) + errlist = validate_and_digest(params) + if errlist: + errmsg = "Invalid format for input parameters" + module.fail_json(msg=errmsg, warnings=errlist, **result) + + # In check_mode, just perform input validation (above), because + # tuned will not be installed on the remote system + if module.check_mode: + module.exit_json(**result) + elif caught_import_error is not None: + raise caught_import_error # pylint: disable-msg=E0702 + elif caught_name_error is not None: + # name error is usually because tuned module was not imported + # but just in case, report it here + raise caught_name_error # pylint: disable-msg=E0702 + + tuned_config = get_tuned_config() + current_profile = None + tuned_app, errmsg = load_current_profile(tuned_config, TUNED_PROFILE, module) + + if tuned_app is None: + module.fail_json(msg=errmsg, **result) + else: + current_profile = tuned_app.daemon.profile + debug_print_profile(current_profile, module) + errmsg = "" + + result["msg"] = "Kernel settings were updated." + + # apply the given params to the profile - if there are any new items + # the function will return True we set changed = True + changestatus, reboot_required = apply_params_to_profile( + params, current_profile, purge + ) + profile_list = [] + if update_current_profile_and_mode(tuned_app.daemon, profile_list): + # profile or mode changed + if changestatus == NOCHANGES: + changestatus = CHANGES + result["msg"] = "Updated active profile and/or mode." + if changestatus > NOCHANGES: + try: + write_profile(current_profile) + # notify tuned to reload/reapply profile + except TunedException as tex: + module.debug("caught TunedException [{0}]".format(tex)) + errmsg = "Unable to apply tuned settings: {0}".format(tex) + module.fail_json(msg=errmsg, **result) + except IOError as ioe: + module.debug("caught IOError [{0}]".format(ioe)) + errmsg = "Unable to apply tuned settings: {0}".format(ioe) + module.fail_json(msg=errmsg, **result) + result["changed"] = True + else: + result["msg"] = "Kernel settings are up to date." + debug_print_profile(current_profile, module) + result["new_profile"] = profile_to_dict(current_profile) + result["active_profile"] = " ".join(profile_list) + result["reboot_required"] = reboot_required + if reboot_required: + result["msg"] = ( + result["msg"] + " A system reboot is needed to apply the changes." + ) + module.exit_json(**result) + + +def main(): + """ The main function! """ + run_module() + + +if __name__ == "__main__": + main() diff --git a/tasks/main.yml b/tasks/main.yml index 6498bab4..803ace3b 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -130,21 +130,6 @@ __kernel_settings_state_absent else none }}" state: "{{ 'absent' if kernel_settings_scsi_host_alpm == __kernel_settings_state_absent else none }}" - eeepc_she: - - name: "{{ 'she_powersave' if kernel_settings_eeepc_she_powersave - else none }}" - value: "{{ kernel_settings_eeepc_she_powersave - if kernel_settings_eeepc_she_powersave != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_eeepc_she_powersave == - __kernel_settings_state_absent else none }}" - - name: "{{ 'she_normal' if kernel_settings_eeepc_she_normal - else none }}" - value: "{{ kernel_settings_eeepc_she_normal - if kernel_settings_eeepc_she_normal != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_eeepc_she_normal == - __kernel_settings_state_absent else none }}" video: - name: "{{ 'radeon_powersave' if kernel_settings_video_radeon_powersave else none }}" diff --git a/tests/tests_change_settings.yml b/tests/tests_change_settings.yml index 70bb269c..c742d305 100644 --- a/tests/tests_change_settings.yml +++ b/tests/tests_change_settings.yml @@ -47,8 +47,6 @@ kernel_settings_audio_timeout: 10 kernel_settings_audio_reset_controller: 1 kernel_settings_scsi_host_alpm: "min_power" - kernel_settings_eeepc_she_powersave: 2 - kernel_settings_eeepc_she_normal: 1 kernel_settings_video_radeon_powersave: "auto" kernel_settings_usb_autosuspend: 1 From abd52d8abd4e7064fb869b56aa75df75de038b60 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 8 Apr 2021 09:45:23 -0400 Subject: [PATCH 12/29] Fix naming scheme for plugin files --- library/kernel_report.py | 123 ++++- library/kernel_settings.py | 980 +++++++++++++++++++++++++++++---- library/old_kernel_report.py | 67 +++ library/old_kernel_settings.py | 892 ------------------------------ 4 files changed, 1031 insertions(+), 1031 deletions(-) create mode 100644 library/old_kernel_report.py delete mode 100644 library/old_kernel_settings.py diff --git a/library/kernel_report.py b/library/kernel_report.py index 8182286a..3be4d1d5 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -6,9 +6,8 @@ # """ Generate kernel settings facts for a system """ -import os import re -import subprocess as sp +import pyudev from ansible.module_utils.basic import AnsibleModule UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events', 'vm\.drop_caches'] @@ -16,27 +15,102 @@ SYSCTL_DIR = '/proc/sys' SYSFS_DIR = '/sys' -def file_get_contents(filename): - with open(filename) as f: - return f.read().rstrip() - -def settings_walk(dir, unstable): - result = [] - combined_unstable = "(" + ")|(".join(unstable) + ")" - for dirpath, dirs, files in os.walk(dir): - if files: - for file in files: - setting_path = dirpath + "/" + file - if(int(oct(os.stat(setting_path).st_mode)[-3:]) >= 644): - formatted_setting = str(setting_path[len(dir)+1:]).replace("/",".") - if re.match(combined_unstable,formatted_setting) is None: - try: - val = file_get_contents(setting_path) - if val: - result.append({'name': formatted_setting, "value": val}) - except OSError as e: - # read errors occur on some of the 'stable_secret' files - pass +def safe_file_get_contents(filename): + try: + with open(filename) as f: + return f.read().rstrip() + except FileNotFoundError as e: + print('safe_file_get_contents: suppressed exception FileNotFoundError with content \'%s\'' % e) + +def get_sysfs_fields(): + result = {} + result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] + result["kernel_settings_transparent_hugepages_defrag"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0] + + # will collect a list of cpus associated to the template machine, but only use settings from the first one + cpus = pyudev.Context().list_devices(subsystem="cpu") + num_cpus = 0 + + for cpu in cpus: + num_cpus += 1 + if 'kernel_settings_cpu_governor' not in result: + result["kernel_settings_cpu_governor"] = safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path) + result["kernel_settings_sampling_down_factor"] = safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % result['kernel_settings_cpu_governor']) + + print("get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus) + + result["kernel_settings_cpu_min_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct") + result["kernel_settings_cpu_max_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct") + result["kernel_settings_cpu_no_turbo"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo") + + # will collect a list of blocks associated to the template machine, but only use settings from the first one + blocks = pyudev.Context().list_devices(subsystem="block") + num_blocks = 0 + + for block in blocks: + num_blocks += 1 + if 'kernel_settings_disk_elevator' not in result: + result["kernel_settings_disk_elevator"] = re.findall(r'\[(\w+)\]', safe_file_get_contents("%s/queue/scheduler" % block.sys_path)) + result["kernel_settings_disk_read_ahead_kb"] = int(safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)) + result["kernel_settings_disk_scheduler_quantum"] = safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path) + + print("get_sysfs_fields: found %d blocks associated to the template machine" % num_blocks) + + result["kernel_settings_avc_cache_threshold"] = int(safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")) + result["kernel_settings_nf_conntrack_hashsize"] = int(safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize")) + + # will collect a list of modules associated to the template machine, but only use settings from the first one + num_modules = 0 + devices = pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") + + for device in devices: + module_name = device.parent.driver + if module_name in ["snd_hda_intel", "snd_ac97_codec"]: + num_modules += 1 + result["kernel_settings_audio_timeout"] = safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name) + result["kernel_settings_audio_reset_controller"] = safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name) + + print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_modules) + + # will collect a list of scsis associated to the template machine, but only use settings from the first one + num_scsis = 0 + scsis = pyudev.Context().list_devices(subsystem="scsi") + + for scsi in scsis: + num_scsis += 1 + if "kernel_settings_scsi_host_alpm" not in result: + result["kernel_settings_scsi_host_alpm"] = safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path) + + print("get_sysfs_fields: found %d scsis associated to the template machine" % num_scsis) + + # will collect a list of graphics cards associated to the template machine, but only use settings from the first one + num_gcards = 0 + gcards = pyudev.Context().list_devices(subsystem="drm").match_sys_name("card*").match_property("DEVTYPE", "drm_minor") + + for gcard in gcards: + num_gcards += 1 + method = safe_file_get_contents("%s/device/power_method" % gcard.sys_path) + if "kernel_settings_video_radeon_powersave" not in result: + if method == "profile": + result["kernel_settings_video_radeon_powersave"] = safe_file_get_contents("%s/device/power_profile" % gcard.sys_path) + elif method == "dynpm": + result["kernel_settings_video_radeon_powersave"] = "dynpm" + elif method == "dpm": + result["kernel_settings_video_radeon_powersave"] = "dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path) + + print("get_sysfs_fields: found %d gcards associated to the template machine" % num_gcards) + + # will collect a list of usb interfaces associated to the template machine, but only use settings from the first one + num_usbs = 0 + usbs = pyudev.Context().list_devices(subsystem="usb") + + for usb in usbs: + num_usbs += 1 + if "kernel_settings_usb_autosuspend" not in result: + result["kernel_settings_usb_autosuspend"] = safe_file_get_contents("%s/power/autosuspend" % usb.sys_path) + + print("get_sysfs_fields: found %d usbs associated to the template machine" % num_usbs) + return result def run_module(): @@ -55,8 +129,7 @@ def run_module(): if module.check_mode: module.exit_json(**result) - # result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS)} - result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS), "sysfs":settings_walk(SYSFS_DIR, UNSTABLE_SYSFS_FIELDS)} + result['ansible_facts'] = get_sysfs_fields() module.exit_json(**result) diff --git a/library/kernel_settings.py b/library/kernel_settings.py index 3be4d1d5..4aa7b02a 100644 --- a/library/kernel_settings.py +++ b/library/kernel_settings.py @@ -1,140 +1,892 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -# Copyright: (c) 2021, Mary Provencher +# Copyright: (c) 2020, Rich Megginson # SPDX-License-Identifier: GPL-2.0-or-later # -""" Generate kernel settings facts for a system """ +""" Manage kernel settings using tuned via a wrapper """ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + +DOCUMENTATION = """ +--- +module: kernel_settings + +short_description: Manage kernel settings using tuned via a wrapper + +version_added: "2.8" + +description: + - | + Manage kernel settings using tuned via a wrapper. The options correspond + to names of units or plugins in tuned. For example, the option C(sysctl) + corresponds to the C(sysctl) unit or plugin in tuned. Setting parameters + works mostly like it does with tuned, except that this module uses Ansible + YAML format instead of the tuned INI-style profile file format. This module + creates a special tuned profile C(kernel_settings) which will be applied by + tuned before any other profiles, allowing the user to configure tuned to + override settings made by this module. You should be aware of this if you + plan to use tuned in addition to using this module. + - HORIZONTALLINE + - | + NOTE: the options list may be incomplete - the actual options are generated + dynamically from tuned, for the current options supported by the version + of tuned, which are the tuned supported plugins. Only the most common + options are listed. See the tuned documentation for the full list and + more information. + - HORIZONTALLINE + - | + Each option takes a list or dict of settings. Each setting is a C(dict). + The C(dict) must have one of the keys C(name), C(value), C(state), or C(previous). + C(state) is used to remove settings or sections of settings. C(previous) is + used to replace all of values in a section with the given values. The only case + where an option takes a dict is when you want to remove a section completely - + then value for the section is the dict C({"state":"empty"}). + If you specify multiple settings with the same name in a section, the last one + will be used. +options: + sysctl: + description: + - | + list of sysctl settings to apply - this works mostly + like the C(sysctl) module except that C(/etc/sysctl.conf) and files + under C(/etc/sysctl.d) are not used. + required: false + sysfs: + description: + - key/value pairs of sysfs settings to apply + required: false + bootloader: + description: + - the C(cmdline) option can be used to set, add, or delete + kernel command line options. See EXAMPLES for some examples + of how to use this option. + Note that this uses the tuned implementation, which adds these + options to whatever the default bootloader command line arguments + tuned is historically used to add/delete performance related + kernel command line arguments e.g. C(spectre_v2=off). If you need + more general purpose bootloader configuration, you should use + a bootloader module/role. + type: raw + purge: + description: + - Remove the current kernel_settings, whatever they are, and force + the given settings to be the current and only settings + type: bool + default: false + +author: + - Rich Megginson (@richm) +""" + +EXAMPLES = """ +# Add or replace the sysctl `fs.file-max` parameter with the value 65535. The +# existing settings are not touched. +- name: Add or replace the sysctl `fs.file-max` parameter with the value 65535. + kernel_settings: + sysctl: + - name: fs.file-max + value: 65535 + +- name: remove the entire sysctl section + kernel_settings: + sysctl: + state: empty + +- name: remove the entire sysctl section and replace with the given values + kernel_settings: + sysctl: + - previous: replaced + - name: fs.file-max + value: 65535 + +- name: add the sysctl vm.max_mmap_regions, and disable spectre/meltdown security + kernel_settings: + sysctl: + - name: vm.max_mmap_regions + value: 262144 + sysfs: + - name: /sys/kernel/debug/x86/pti_enabled + value: 0 + - name: /sys/kernel/debug/x86/retp_enabled + value: 0 + - name: /sys/kernel/debug/x86/ibrs_enabled + value: 0 + +# replace the existing sysctl section with the specified section +# delete the /sys/kernel/debug/x86/retp_enabled setting +# completely remove the vm section +# add the bootloader cmdline arguments spectre_v2=off nopti +# remove the bootloader cmdline arguments panic splash +- name: more settings + kernel_settings: + sysctl: + - previous: replaced + - name: vm.max_mmap_regions + value: 262144 + sysfs: + - name: /sys/kernel/debug/x86/retp_enabled + state: absent + vm: + state: empty + bootloader: + - name: cmdline + value: + - name: spectre_v2 + value: off + - name: nopti + - name: panic + state: absent + - name: splash + state: absent +""" + +RETURN = """ +msg: + description: | + A short text message to say what action this module performed. + returned: always + type: str +new_profile: + description: | + This is the tuned profile in dict format, after the changes, if any, + have been applied. + returned: always + type: dict +reboot_required: + description: | + default C(false) - if true, this means a reboot of the managed host is + required in order for the changes to take effect. + returned: always + type: bool +active_profile: + description: | + This is the space delimited list of active profiles as reported + by tuned. + returned: always + type: str +""" + +import os +import logging import re -import pyudev +import tempfile +import shutil +import shlex +import contextlib +import atexit # for testing +import copy + +try: + import configobj + + HAS_CONFIGOBJ = True +except ImportError: + HAS_CONFIGOBJ = False +import ansible.module_utils.six as ansible_six + +# This is a bit of a mystery - bug in pylint? +# pylint: disable=import-error +import ansible.module_utils.six.moves as ansible_six_moves + from ansible.module_utils.basic import AnsibleModule -UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events', 'vm\.drop_caches'] -UNSTABLE_SYSFS_FIELDS = ['kernel\.debug', 'devices'] -SYSCTL_DIR = '/proc/sys' -SYSFS_DIR = '/sys' +# see https://github.com/python/cpython/blob/master/Lib/logging/__init__.py +# for information about logging module internals + + +class TunedLogWrapper(logging.getLoggerClass()): + """This wraps the tuned logger so that we can intercept logs and handle them here""" + + def __init__(self, *args, **kwargs): + super(TunedLogWrapper, self).__init__(*args, **kwargs) + self.setLevel(logging.DEBUG) + self.logstack = [] + + def handle(self, record): + self.logstack.append(record) + + +@contextlib.contextmanager +def save_and_restore_logging_methods(): + """do not allow tuned logging to pollute global logging module or + ansible logging""" + save_logging_add_level_name = logging.addLevelName -def safe_file_get_contents(filename): + def wrapper_add_level_name(_levelval, _levelname): + """ignore tuned.logging call to logging.addLevelName""" + # print('addLevelName wrapper ignoring {} {}'.format(levelval, levelname)) + + logging.addLevelName = wrapper_add_level_name + save_logging_set_logger_class = logging.setLoggerClass + + def wrapper_set_logger_class(_clsname): + """ignore tuned.logging call to logging.setLoggerClass""" + # print('setLoggerClass wrapper ignoring {}'.format(clsname)) + + logging.setLoggerClass = wrapper_set_logger_class try: - with open(filename) as f: - return f.read().rstrip() - except FileNotFoundError as e: - print('safe_file_get_contents: suppressed exception FileNotFoundError with content \'%s\'' % e) - -def get_sysfs_fields(): - result = {} - result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] - result["kernel_settings_transparent_hugepages_defrag"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0] - - # will collect a list of cpus associated to the template machine, but only use settings from the first one - cpus = pyudev.Context().list_devices(subsystem="cpu") - num_cpus = 0 - - for cpu in cpus: - num_cpus += 1 - if 'kernel_settings_cpu_governor' not in result: - result["kernel_settings_cpu_governor"] = safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path) - result["kernel_settings_sampling_down_factor"] = safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % result['kernel_settings_cpu_governor']) - - print("get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus) - - result["kernel_settings_cpu_min_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct") - result["kernel_settings_cpu_max_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct") - result["kernel_settings_cpu_no_turbo"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo") - - # will collect a list of blocks associated to the template machine, but only use settings from the first one - blocks = pyudev.Context().list_devices(subsystem="block") - num_blocks = 0 - - for block in blocks: - num_blocks += 1 - if 'kernel_settings_disk_elevator' not in result: - result["kernel_settings_disk_elevator"] = re.findall(r'\[(\w+)\]', safe_file_get_contents("%s/queue/scheduler" % block.sys_path)) - result["kernel_settings_disk_read_ahead_kb"] = int(safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)) - result["kernel_settings_disk_scheduler_quantum"] = safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path) - - print("get_sysfs_fields: found %d blocks associated to the template machine" % num_blocks) - - result["kernel_settings_avc_cache_threshold"] = int(safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")) - result["kernel_settings_nf_conntrack_hashsize"] = int(safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize")) - - # will collect a list of modules associated to the template machine, but only use settings from the first one - num_modules = 0 - devices = pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") - - for device in devices: - module_name = device.parent.driver - if module_name in ["snd_hda_intel", "snd_ac97_codec"]: - num_modules += 1 - result["kernel_settings_audio_timeout"] = safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name) - result["kernel_settings_audio_reset_controller"] = safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name) - - print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_modules) - - # will collect a list of scsis associated to the template machine, but only use settings from the first one - num_scsis = 0 - scsis = pyudev.Context().list_devices(subsystem="scsi") - - for scsi in scsis: - num_scsis += 1 - if "kernel_settings_scsi_host_alpm" not in result: - result["kernel_settings_scsi_host_alpm"] = safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path) - - print("get_sysfs_fields: found %d scsis associated to the template machine" % num_scsis) - - # will collect a list of graphics cards associated to the template machine, but only use settings from the first one - num_gcards = 0 - gcards = pyudev.Context().list_devices(subsystem="drm").match_sys_name("card*").match_property("DEVTYPE", "drm_minor") - - for gcard in gcards: - num_gcards += 1 - method = safe_file_get_contents("%s/device/power_method" % gcard.sys_path) - if "kernel_settings_video_radeon_powersave" not in result: - if method == "profile": - result["kernel_settings_video_radeon_powersave"] = safe_file_get_contents("%s/device/power_profile" % gcard.sys_path) - elif method == "dynpm": - result["kernel_settings_video_radeon_powersave"] = "dynpm" - elif method == "dpm": - result["kernel_settings_video_radeon_powersave"] = "dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path) - - print("get_sysfs_fields: found %d gcards associated to the template machine" % num_gcards) - - # will collect a list of usb interfaces associated to the template machine, but only use settings from the first one - num_usbs = 0 - usbs = pyudev.Context().list_devices(subsystem="usb") - - for usb in usbs: - num_usbs += 1 - if "kernel_settings_usb_autosuspend" not in result: - result["kernel_settings_usb_autosuspend"] = safe_file_get_contents("%s/power/autosuspend" % usb.sys_path) - - print("get_sysfs_fields: found %d usbs associated to the template machine" % num_usbs) - - return result + yield + finally: + logging.addLevelName = save_logging_add_level_name + logging.setLoggerClass = save_logging_set_logger_class -def run_module(): - module_args = dict() - result = dict( - changed=False, - ansible_facts=dict(), +caught_import_error = None +HAS_TUNED = False +try: + with save_and_restore_logging_methods(): + import tuned.logs + HAS_TUNED = True +except ImportError as ierr: + # tuned package might not be available in check mode - so just + # note that this is missing, and do not report in check mode + caught_import_error = ierr + +if HAS_TUNED: + tuned.logs.root_logger = TunedLogWrapper(__name__) + tuned.logs.get = lambda: tuned.logs.root_logger + + import tuned.consts + import tuned.utils.global_config + import tuned.daemon + from tuned.exceptions import TunedException + + +TUNED_PROFILE = os.environ.get("TEST_PROFILE", "kernel_settings") +NOCHANGES = 0 +CHANGES = 1 +REMOVE_SECTION_VALUE = {"state": "empty"} +SECTION_TO_REPLACE = "__section_to_replace" +ERR_SECTION_MISSING_NAME = "Error: section [{0}] item is missing 'name': {1}" +ERR_NAME_NOT_VALID = "Error: section [{0}] item name [{1}] is not a valid string" +ERR_NO_VALUE_OR_STATE = ( + "Error: section [{0}] item name [{1}] must have either a 'value' or 'state'" +) +ERR_BOTH_VALUE_AND_STATE = ( + "Error: section [{0}] item name [{1}] must have only one of 'value' or 'state'" +) +ERR_STATE_ABSENT = ( + "Error: section [{0}] item name [{1}] state value must be 'absent' not [{2}]" +) +ERR_UNEXPECTED_VALUES = "Error: section [{0}] item [{1}] has unexpected values {2}" +ERR_VALUE_ITEM_NOT_DICT = ( + "Error: section [{0}] item name [{1}] value [{2}] is not a dict" +) +ERR_VALUE_ITEM_PREVIOUS = ( + "Error: section [{0}] item name [{1}] has invalid value for 'previous' [{2}]" +) +ERR_REMOVE_SECTION_VALUE = "Error: to remove the section [{0}] specify the value {1}" +ERR_ITEM_NOT_DICT = "Error: section [{0}] item value [{1}] is not a dict" +ERR_ITEM_PREVIOUS = "Error: section [{0}] item has invalid value for 'previous' [{1}]" +ERR_ITEM_DICT_OR_LIST = "Error: section [{0}] value must be a dict or a list" +ERR_LIST_NOT_ALLOWED = "Error: section [{0}] item [{1}] has unexpected list value {2}" +ERR_BLCMD_MUST_BE_LIST = "Error: section [{0}] item [{1}] must be a list not [{2}]" +ERR_VALUE_CANNOT_BE_BOOLEAN = ( + "Error: section [{0}] item [{1}] value [{2}] must not " + "be a boolean - try quoting the value" +) + + +def get_supported_tuned_plugin_names(): + """get names of all tuned plugins supported by this module""" + return [ + "bootloader", + "modules", + "selinux", + "sysctl", + "sysfs", + "systemd", + "vm", + "cpu", + "disk", + "net", + "audio", + "scsi_host", + "video", + "usb", + ] + + +def no_such_profile(): + """see if the last log message was that the profile did not exist""" + lastlogmsg = tuned.logs.root_logger.logstack[-1].msg + return re.search("Requested profile .* doesn't exist", lastlogmsg) is not None + + +def profile_to_dict(profile): + """convert profile object to dict""" + ret_val = {} + for unitname, unit in profile.units.items(): + ret_val[unitname] = unit.options + return ret_val + + +def debug_print_profile(profile, amodule): + """for debugging - print profile as INI""" + amodule.debug("profile {0}".format(profile.name)) + amodule.debug(str(profile_to_dict(profile))) + + +caught_name_error = None +try: + EMPTYUNIT = tuned.profiles.unit.Unit("empty", {}) +except NameError as nerr: + # tuned not loaded in check mode + caught_name_error = nerr + + +def get_profile_unit_key(profile, unitname, key): + """convenience function""" + return profile.units.get(unitname, EMPTYUNIT).options.get(key) + + +class BLCmdLine(object): + """A data type for handling bootloader cmdline values.""" + + def __init__(self, val): + self.key_list = [] # list of keys in order - to preserve ordering + self.key_to_val = {} # maps key to value + if val: + for item in shlex.split(val): + key, val = self.splititem(item) + self.key_list.append(key) + # None or '' means bare key - no value + self.key_to_val[key] = val + + @classmethod + def splititem(cls, item): + """split item in form key=somevalue into key and somevalue""" + # pylint: disable=blacklisted-name + key, _, val = item.partition("=") + return key, val + + @classmethod + def escapeval(cls, val): + """make sure val is quoted as in shell""" + return ansible_six_moves.shlex_quote(str(val)) + + def __str__(self): + vallist = [] + for key in self.key_list: + val = self.key_to_val[key] + if val: + vallist.append("{0}={1}".format(key, self.escapeval(val))) + else: + vallist.append(key) + return " ".join(vallist) + + def add(self, key, val): + """add/replace the given key & value""" + if key not in self.key_to_val: + self.key_list.append(key) + self.key_to_val[key] = val + + def remove(self, key): + """remove the given key""" + if key in self.key_to_val: + self.key_list.remove(key) + del self.key_to_val[key] + + +def apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue): + """apply a bootloader cmdline item to the current profile""" + name = item["name"] + if item.get(SECTION_TO_REPLACE, False): + blcmd = BLCmdLine("") + else: + blcmd = BLCmdLine(curvalue) + for subitem in item["value"]: + if "previous" in subitem: + continue + if subitem.get("state") == "absent": + blcmd.remove(subitem["name"]) + else: + blcmd.add(subitem["name"], subitem.get("value")) + blcmdstr = str(blcmd) + if blcmdstr: + current_profile.units.setdefault( + unitname, tuned.profiles.unit.Unit(unitname, {}) + ).options[name] = blcmdstr + elif curvalue: + del current_profile.units[unitname].options[name] + + +def apply_item_to_profile(item, unitname, current_profile): + """apply a specific item from a section to the current_profile""" + name = item["name"] + curvalue = get_profile_unit_key(current_profile, unitname, name) + newvalue = item.get("value") + if item.get("state", None) == "absent": + if curvalue: + del current_profile.units[unitname].options[name] + elif unitname == "bootloader" and name == "cmdline": + apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue) + else: + current_profile.units.setdefault( + unitname, tuned.profiles.unit.Unit(unitname, {}) + ).options[name] = str(newvalue) + + +def is_reboot_required(unitname): + """Some changes need a reboot for the changes to be applied + For example, bootloader cmdline changes""" + # for now, only bootloader cmdline changes need a reboot + return unitname == "bootloader" + + +def apply_params_to_profile(params, current_profile, purge): + """apply the settings from the input parameters to the current profile + delete the unit if it is empty after applying the parameter deletions + """ + changestatus = NOCHANGES + reboot_required = False + section_to_replace = params.pop(SECTION_TO_REPLACE, {}) + need_purge = set() + if purge: + # remove units not specified in params + need_purge = set(current_profile.units.keys()) + for unitname, items in params.items(): + unit = current_profile.units.get(unitname, None) + current_options = {} + if unit: + current_options = copy.deepcopy(unit.options) + replace = section_to_replace.get(unitname, purge) + if replace or (items == REMOVE_SECTION_VALUE): + if unit: + unit.options.clear() + if purge and unitname in need_purge: + need_purge.remove(unitname) + if items == REMOVE_SECTION_VALUE: + if unit: + # we changed the profile + changestatus = CHANGES + reboot_required = reboot_required or is_reboot_required(unitname) + # we're done - no further processing necessary for this unit + continue + for item in items: + if item and "previous" not in item: + apply_item_to_profile(item, unitname, current_profile) + newoptions = {} + if unitname in current_profile.units: + newoptions = current_profile.units[unitname].options + if current_options != newoptions: + changestatus = CHANGES + reboot_required = reboot_required or is_reboot_required(unitname) + for unitname in need_purge: + del current_profile.units[unitname] + changestatus = CHANGES + reboot_required = reboot_required or is_reboot_required(unitname) + # remove empty units + for unitname in list(current_profile.units.keys()): + if not current_profile.units[unitname].options: + del current_profile.units[unitname] + return changestatus, reboot_required + + +def write_profile(current_profile): + """write the profile to the profile file""" + # convert profile to configobj to write ini-style file + # profile.options go into [main] section + # profile.units go into [unitname] section + cfg = configobj.ConfigObj(indent_type="") + cfg.initial_comment = ["File managed by Ansible - DO NOT EDIT"] + cfg["main"] = current_profile.options + for unitname, unit in current_profile.units.items(): + cfg[unitname] = unit.options + profile_base_dir = tuned.consts.LOAD_DIRECTORIES[-1] + prof_fname = os.path.join( + profile_base_dir, TUNED_PROFILE, tuned.consts.PROFILE_FILE ) + with open(prof_fname, "wb") as prof_f: + cfg.write(prof_f) + - module = AnsibleModule( - argument_spec=module_args, - supports_check_mode=True +def update_current_profile_and_mode(daemon, profile_list): + """ensure that the tuned current_profile applies the kernel_settings last""" + changed = False + profile, manual = daemon._get_startup_profile() + # is TUNED_PROFILE in the list? + profile_list.extend(profile.split()) + if TUNED_PROFILE not in profile_list: + changed = True + profile_list.append(TUNED_PROFILE) + # have to convert to manual mode in order to ensure kernel_settings are applied + if not manual: + changed = True + manual = True + if changed: + daemon._save_active_profile(" ".join(profile_list), manual) + return changed + + +def setup_for_testing(): + """create an /etc/tuned and /usr/lib/tuned directory structure for testing""" + test_root_dir = os.environ.get("TEST_ROOT_DIR") + if test_root_dir is None: + test_root_dir = tempfile.mkdtemp(suffix=".lsr") + # copy all of the test configs and profiles + test_root_dir_tuned = os.path.join(test_root_dir, "tuned") + test_src_dir = os.environ.get("TEST_SRC_DIR", "tests") + src_dir = os.path.join(test_src_dir, "tuned") + shutil.copytree(src_dir, test_root_dir_tuned) + # patch all of the consts to use the test_root_dir + orig_consts = {} + for cnst in ( + "GLOBAL_CONFIG_FILE", + "ACTIVE_PROFILE_FILE", + "PROFILE_MODE_FILE", + "RECOMMEND_CONF_FILE", + "BOOT_CMDLINE_FILE", + ): + orig_consts[cnst] = tuned.consts.__dict__[cnst] + fname = os.path.join( + test_root_dir_tuned, + os.path.relpath(tuned.consts.__dict__[cnst], os.path.sep), + ) + tuned.consts.__dict__[cnst] = fname + dname = os.path.dirname(fname) + if not os.path.isdir(dname): + os.makedirs(dname) + orig_load_dirs = [] + for idx, dname in enumerate(tuned.consts.LOAD_DIRECTORIES): + orig_load_dirs.append(dname) + newdname = os.path.join( + test_root_dir_tuned, os.path.relpath(dname, os.path.sep) + ) + tuned.consts.LOAD_DIRECTORIES[idx] = newdname + if not os.path.isdir(newdname): + os.makedirs(newdname) + orig_rec_dirs = [] + for idx, dname in enumerate(tuned.consts.RECOMMEND_DIRECTORIES): + orig_rec_dirs.append(dname) + newdname = os.path.join( + test_root_dir_tuned, os.path.relpath(dname, os.path.sep) + ) + tuned.consts.RECOMMEND_DIRECTORIES[idx] = newdname + if not os.path.isdir(newdname): + os.makedirs(newdname) + has_func = bool( + getattr(tuned.utils.global_config.GlobalConfig.__init__, "__func__", False) ) + if has_func: + orig_gc_init_defaults = ( + tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ + ) + orig_gc_load_config_defaults = ( + tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ + ) + if orig_gc_init_defaults: + tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + if orig_gc_load_config_defaults: + tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + else: + orig_gc_init_defaults = ( + tuned.utils.global_config.GlobalConfig.__init__.__defaults__ + ) + orig_gc_load_config_defaults = ( + tuned.utils.global_config.GlobalConfig.load_config.__defaults__ + ) + if orig_gc_init_defaults: + tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + if orig_gc_load_config_defaults: + tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( + tuned.consts.GLOBAL_CONFIG_FILE, + ) + # this call fails on ubuntu and containers, so mock it for testing + import pyudev.monitor + + orig_set_receive_buffer_size = pyudev.monitor.Monitor.set_receive_buffer_size + pyudev.monitor.Monitor.set_receive_buffer_size = lambda self, size: None + + def test_cleanup(): + import os + import shutil + + if "TEST_ROOT_DIR" not in os.environ: + shutil.rmtree(test_root_dir) + for cnst, val in orig_consts.items(): + tuned.consts.__dict__[cnst] = val + for idx, dname in enumerate(orig_load_dirs): + tuned.consts.LOAD_DIRECTORIES[idx] = dname + for idx, dname in enumerate(orig_rec_dirs): + tuned.consts.RECOMMEND_DIRECTORIES[idx] = dname + if has_func: + tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( + orig_gc_init_defaults + ) + tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( + orig_gc_load_config_defaults + ) + else: + tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( + orig_gc_init_defaults + ) + tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( + orig_gc_load_config_defaults + ) + pyudev.monitor.Monitor.set_receive_buffer_size = orig_set_receive_buffer_size + + if "TEST_ROOT_DIR" not in os.environ: + atexit.register(test_cleanup) + return test_cleanup + + +def get_tuned_config(): + """get the tuned config and set our parameters in it""" + tuned_config = tuned.utils.global_config.GlobalConfig() + tuned_config.set("daemon", 0) + tuned_config.set("reapply_sysctl", 0) + tuned_config.set("dynamic_tuning", 0) + return tuned_config + +def load_current_profile(tuned_config, tuned_profile, logger): + """load the current profile""" + tuned_app = None + errmsg = "Error loading tuned profile [{0}]".format(TUNED_PROFILE) + try: + tuned_app = tuned.daemon.Application(tuned_profile, tuned_config) + except TunedException as tex: + logger.debug("caught TunedException [{0}]".format(tex)) + errmsg = errmsg + ": {0}".format(tex) + tuned_app = None + except IOError as ioe: + # for testing this case, need to create a profile with a bad permissions e.g. + # mkdir ioerror_profile; touch ioerror_profile/tuned.conf; chmod 0000 !$ + logger.debug("caught IOError [{0}]".format(ioe)) + errmsg = errmsg + ": {0}".format(ioe) + tuned_app = None + if ( + tuned_app is None + or tuned_app.daemon is None + or tuned_app.daemon.profile is None + or tuned_app.daemon.profile.units is None + or tuned_app.daemon.profile.options is None + or "summary" not in tuned_app.daemon.profile.options + ): + tuned_app = None + if no_such_profile(): + errmsg = errmsg + ": Profile does not exist" + return tuned_app, errmsg + + +def validate_and_digest_item(sectionname, item, listallowed=True, allowempty=False): + """Validate an item - must contain only name, value, and state""" + tmpitem = item.copy() + name = tmpitem.pop("name", None) + value = tmpitem.pop("value", None) + state = tmpitem.pop("state", None) + errlist = [] + if name is None: + errlist.append(ERR_SECTION_MISSING_NAME.format(sectionname, item)) + elif not isinstance(name, ansible_six.string_types): + errlist.append(ERR_NAME_NOT_VALID.format(sectionname, name)) + elif (value is None and not allowempty) and state is None: + errlist.append(ERR_NO_VALUE_OR_STATE.format(sectionname, name)) + elif value is not None and state is not None: + errlist.append(ERR_BOTH_VALUE_AND_STATE.format(sectionname, name)) + elif state is not None and state != "absent": + errlist.append(ERR_STATE_ABSENT.format(sectionname, name, state)) + elif tmpitem: + errlist.append(ERR_UNEXPECTED_VALUES.format(sectionname, name, tmpitem)) + elif isinstance(value, list): + if not listallowed: + errlist.append(ERR_LIST_NOT_ALLOWED.format(sectionname, name, value)) + else: + for valitem in value: + if not isinstance(valitem, dict): + errlist.append( + ERR_VALUE_ITEM_NOT_DICT.format(sectionname, name, valitem) + ) + elif "previous" in valitem: + if valitem["previous"] != "replaced": + errlist.append( + ERR_VALUE_ITEM_PREVIOUS.format( + sectionname, name, valitem["previous"] + ) + ) + else: + item[SECTION_TO_REPLACE] = True + else: + tmperrlist = validate_and_digest_item( + sectionname, valitem, False, True + ) + errlist.extend(tmperrlist) + elif sectionname == "bootloader" and name == "cmdline": + errlist.append(ERR_BLCMD_MUST_BE_LIST.format(sectionname, name, value)) + elif isinstance(value, bool): + errlist.append(ERR_VALUE_CANNOT_BE_BOOLEAN.format(sectionname, name, value)) + return errlist + + +def validate_and_digest(params): + """Validate that params is in the correct format, since we + are using type `raw`, we have to perform the validation here. + Also do some pre-processing to make it easier to apply + the params to profile""" + errlist = [] + replaces = {} + for sectionname, items in params.items(): + if isinstance(items, dict): + if not items == REMOVE_SECTION_VALUE: + errlist.append( + ERR_REMOVE_SECTION_VALUE.format(sectionname, REMOVE_SECTION_VALUE) + ) + elif isinstance(items, list): + for item in items: + if not isinstance(item, dict): + errlist.append(ERR_ITEM_NOT_DICT.format(sectionname, item)) + elif not item: + continue # ignore empty items + elif "previous" in item: + if item["previous"] != "replaced": + errlist.append( + ERR_ITEM_PREVIOUS.format(sectionname, item["previous"]) + ) + else: + replaces[sectionname] = True + else: + itemerrlist = validate_and_digest_item(sectionname, item) + errlist.extend(itemerrlist) + else: + errlist.append(ERR_ITEM_DICT_OR_LIST.format(sectionname)) + if replaces: + params[SECTION_TO_REPLACE] = replaces + + return errlist + + +def remove_if_empty(params): + """recursively remove empty items from params + return true if params results in being empty, + false otherwise""" + if isinstance(params, list): + removed = 0 + for idx in range(0, len(params)): + realidx = idx - removed + if remove_if_empty(params[realidx]): + del params[realidx] + removed = removed + 1 + elif isinstance(params, dict): + for key, val in list(params.items()): + if remove_if_empty(val): + del params[key] + return params == [] or params == {} or params == "" or params is None + + +def run_module(): + """ The entry point of the module. """ + + module_args = dict( + purge=dict(type="bool", required=False, default=False), + ) + tuned_plugin_names = get_supported_tuned_plugin_names() + for plugin_name in tuned_plugin_names: + # use raw here - type can be dict or list - perform validation + # below + module_args[plugin_name] = dict(type="raw", required=False) + + result = dict(changed=False, message="") + + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) + + if not module.check_mode: + if os.environ.get("TESTING", "false") == "true": + # pylint: disable=blacklisted-name + _ = setup_for_testing() + + params = module.params + # remove any non-tuned fields from params and save them locally + # state = params.pop("state") + purge = params.pop("purge", False) + # also remove any empty or None + # pylint: disable=blacklisted-name + _ = remove_if_empty(params) + errlist = validate_and_digest(params) + if errlist: + errmsg = "Invalid format for input parameters" + module.fail_json(msg=errmsg, warnings=errlist, **result) + + # In check_mode, just perform input validation (above), because + # tuned will not be installed on the remote system if module.check_mode: module.exit_json(**result) + elif caught_import_error is not None: + raise caught_import_error # pylint: disable-msg=E0702 + elif caught_name_error is not None: + # name error is usually because tuned module was not imported + # but just in case, report it here + raise caught_name_error # pylint: disable-msg=E0702 - result['ansible_facts'] = get_sysfs_fields() + tuned_config = get_tuned_config() + current_profile = None + tuned_app, errmsg = load_current_profile(tuned_config, TUNED_PROFILE, module) + if tuned_app is None: + module.fail_json(msg=errmsg, **result) + else: + current_profile = tuned_app.daemon.profile + debug_print_profile(current_profile, module) + errmsg = "" + + result["msg"] = "Kernel settings were updated." + + # apply the given params to the profile - if there are any new items + # the function will return True we set changed = True + changestatus, reboot_required = apply_params_to_profile( + params, current_profile, purge + ) + profile_list = [] + if update_current_profile_and_mode(tuned_app.daemon, profile_list): + # profile or mode changed + if changestatus == NOCHANGES: + changestatus = CHANGES + result["msg"] = "Updated active profile and/or mode." + if changestatus > NOCHANGES: + try: + write_profile(current_profile) + # notify tuned to reload/reapply profile + except TunedException as tex: + module.debug("caught TunedException [{0}]".format(tex)) + errmsg = "Unable to apply tuned settings: {0}".format(tex) + module.fail_json(msg=errmsg, **result) + except IOError as ioe: + module.debug("caught IOError [{0}]".format(ioe)) + errmsg = "Unable to apply tuned settings: {0}".format(ioe) + module.fail_json(msg=errmsg, **result) + result["changed"] = True + else: + result["msg"] = "Kernel settings are up to date." + debug_print_profile(current_profile, module) + result["new_profile"] = profile_to_dict(current_profile) + result["active_profile"] = " ".join(profile_list) + result["reboot_required"] = reboot_required + if reboot_required: + result["msg"] = ( + result["msg"] + " A system reboot is needed to apply the changes." + ) module.exit_json(**result) + def main(): + """ The main function! """ run_module() -if __name__ == '__main__': - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/library/old_kernel_report.py b/library/old_kernel_report.py new file mode 100644 index 00000000..8182286a --- /dev/null +++ b/library/old_kernel_report.py @@ -0,0 +1,67 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2021, Mary Provencher +# SPDX-License-Identifier: GPL-2.0-or-later +# +""" Generate kernel settings facts for a system """ + +import os +import re +import subprocess as sp +from ansible.module_utils.basic import AnsibleModule + +UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events', 'vm\.drop_caches'] +UNSTABLE_SYSFS_FIELDS = ['kernel\.debug', 'devices'] +SYSCTL_DIR = '/proc/sys' +SYSFS_DIR = '/sys' + +def file_get_contents(filename): + with open(filename) as f: + return f.read().rstrip() + +def settings_walk(dir, unstable): + result = [] + combined_unstable = "(" + ")|(".join(unstable) + ")" + for dirpath, dirs, files in os.walk(dir): + if files: + for file in files: + setting_path = dirpath + "/" + file + if(int(oct(os.stat(setting_path).st_mode)[-3:]) >= 644): + formatted_setting = str(setting_path[len(dir)+1:]).replace("/",".") + if re.match(combined_unstable,formatted_setting) is None: + try: + val = file_get_contents(setting_path) + if val: + result.append({'name': formatted_setting, "value": val}) + except OSError as e: + # read errors occur on some of the 'stable_secret' files + pass + return result + +def run_module(): + module_args = dict() + + result = dict( + changed=False, + ansible_facts=dict(), + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + if module.check_mode: + module.exit_json(**result) + + # result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS)} + result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS), "sysfs":settings_walk(SYSFS_DIR, UNSTABLE_SYSFS_FIELDS)} + + module.exit_json(**result) + +def main(): + run_module() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/library/old_kernel_settings.py b/library/old_kernel_settings.py deleted file mode 100644 index 4aa7b02a..00000000 --- a/library/old_kernel_settings.py +++ /dev/null @@ -1,892 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright: (c) 2020, Rich Megginson -# SPDX-License-Identifier: GPL-2.0-or-later -# -""" Manage kernel settings using tuned via a wrapper """ - -from __future__ import absolute_import, division, print_function - -__metaclass__ = type - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "community", -} - -DOCUMENTATION = """ ---- -module: kernel_settings - -short_description: Manage kernel settings using tuned via a wrapper - -version_added: "2.8" - -description: - - | - Manage kernel settings using tuned via a wrapper. The options correspond - to names of units or plugins in tuned. For example, the option C(sysctl) - corresponds to the C(sysctl) unit or plugin in tuned. Setting parameters - works mostly like it does with tuned, except that this module uses Ansible - YAML format instead of the tuned INI-style profile file format. This module - creates a special tuned profile C(kernel_settings) which will be applied by - tuned before any other profiles, allowing the user to configure tuned to - override settings made by this module. You should be aware of this if you - plan to use tuned in addition to using this module. - - HORIZONTALLINE - - | - NOTE: the options list may be incomplete - the actual options are generated - dynamically from tuned, for the current options supported by the version - of tuned, which are the tuned supported plugins. Only the most common - options are listed. See the tuned documentation for the full list and - more information. - - HORIZONTALLINE - - | - Each option takes a list or dict of settings. Each setting is a C(dict). - The C(dict) must have one of the keys C(name), C(value), C(state), or C(previous). - C(state) is used to remove settings or sections of settings. C(previous) is - used to replace all of values in a section with the given values. The only case - where an option takes a dict is when you want to remove a section completely - - then value for the section is the dict C({"state":"empty"}). - If you specify multiple settings with the same name in a section, the last one - will be used. -options: - sysctl: - description: - - | - list of sysctl settings to apply - this works mostly - like the C(sysctl) module except that C(/etc/sysctl.conf) and files - under C(/etc/sysctl.d) are not used. - required: false - sysfs: - description: - - key/value pairs of sysfs settings to apply - required: false - bootloader: - description: - - the C(cmdline) option can be used to set, add, or delete - kernel command line options. See EXAMPLES for some examples - of how to use this option. - Note that this uses the tuned implementation, which adds these - options to whatever the default bootloader command line arguments - tuned is historically used to add/delete performance related - kernel command line arguments e.g. C(spectre_v2=off). If you need - more general purpose bootloader configuration, you should use - a bootloader module/role. - type: raw - purge: - description: - - Remove the current kernel_settings, whatever they are, and force - the given settings to be the current and only settings - type: bool - default: false - -author: - - Rich Megginson (@richm) -""" - -EXAMPLES = """ -# Add or replace the sysctl `fs.file-max` parameter with the value 65535. The -# existing settings are not touched. -- name: Add or replace the sysctl `fs.file-max` parameter with the value 65535. - kernel_settings: - sysctl: - - name: fs.file-max - value: 65535 - -- name: remove the entire sysctl section - kernel_settings: - sysctl: - state: empty - -- name: remove the entire sysctl section and replace with the given values - kernel_settings: - sysctl: - - previous: replaced - - name: fs.file-max - value: 65535 - -- name: add the sysctl vm.max_mmap_regions, and disable spectre/meltdown security - kernel_settings: - sysctl: - - name: vm.max_mmap_regions - value: 262144 - sysfs: - - name: /sys/kernel/debug/x86/pti_enabled - value: 0 - - name: /sys/kernel/debug/x86/retp_enabled - value: 0 - - name: /sys/kernel/debug/x86/ibrs_enabled - value: 0 - -# replace the existing sysctl section with the specified section -# delete the /sys/kernel/debug/x86/retp_enabled setting -# completely remove the vm section -# add the bootloader cmdline arguments spectre_v2=off nopti -# remove the bootloader cmdline arguments panic splash -- name: more settings - kernel_settings: - sysctl: - - previous: replaced - - name: vm.max_mmap_regions - value: 262144 - sysfs: - - name: /sys/kernel/debug/x86/retp_enabled - state: absent - vm: - state: empty - bootloader: - - name: cmdline - value: - - name: spectre_v2 - value: off - - name: nopti - - name: panic - state: absent - - name: splash - state: absent -""" - -RETURN = """ -msg: - description: | - A short text message to say what action this module performed. - returned: always - type: str -new_profile: - description: | - This is the tuned profile in dict format, after the changes, if any, - have been applied. - returned: always - type: dict -reboot_required: - description: | - default C(false) - if true, this means a reboot of the managed host is - required in order for the changes to take effect. - returned: always - type: bool -active_profile: - description: | - This is the space delimited list of active profiles as reported - by tuned. - returned: always - type: str -""" - -import os -import logging -import re -import tempfile -import shutil -import shlex -import contextlib -import atexit # for testing -import copy - -try: - import configobj - - HAS_CONFIGOBJ = True -except ImportError: - HAS_CONFIGOBJ = False -import ansible.module_utils.six as ansible_six - -# This is a bit of a mystery - bug in pylint? -# pylint: disable=import-error -import ansible.module_utils.six.moves as ansible_six_moves - -from ansible.module_utils.basic import AnsibleModule - -# see https://github.com/python/cpython/blob/master/Lib/logging/__init__.py -# for information about logging module internals - - -class TunedLogWrapper(logging.getLoggerClass()): - """This wraps the tuned logger so that we can intercept logs and handle them here""" - - def __init__(self, *args, **kwargs): - super(TunedLogWrapper, self).__init__(*args, **kwargs) - self.setLevel(logging.DEBUG) - self.logstack = [] - - def handle(self, record): - self.logstack.append(record) - - -@contextlib.contextmanager -def save_and_restore_logging_methods(): - """do not allow tuned logging to pollute global logging module or - ansible logging""" - save_logging_add_level_name = logging.addLevelName - - def wrapper_add_level_name(_levelval, _levelname): - """ignore tuned.logging call to logging.addLevelName""" - # print('addLevelName wrapper ignoring {} {}'.format(levelval, levelname)) - - logging.addLevelName = wrapper_add_level_name - save_logging_set_logger_class = logging.setLoggerClass - - def wrapper_set_logger_class(_clsname): - """ignore tuned.logging call to logging.setLoggerClass""" - # print('setLoggerClass wrapper ignoring {}'.format(clsname)) - - logging.setLoggerClass = wrapper_set_logger_class - try: - yield - finally: - logging.addLevelName = save_logging_add_level_name - logging.setLoggerClass = save_logging_set_logger_class - - -caught_import_error = None -HAS_TUNED = False -try: - with save_and_restore_logging_methods(): - import tuned.logs - HAS_TUNED = True -except ImportError as ierr: - # tuned package might not be available in check mode - so just - # note that this is missing, and do not report in check mode - caught_import_error = ierr - -if HAS_TUNED: - tuned.logs.root_logger = TunedLogWrapper(__name__) - tuned.logs.get = lambda: tuned.logs.root_logger - - import tuned.consts - import tuned.utils.global_config - import tuned.daemon - from tuned.exceptions import TunedException - - -TUNED_PROFILE = os.environ.get("TEST_PROFILE", "kernel_settings") -NOCHANGES = 0 -CHANGES = 1 -REMOVE_SECTION_VALUE = {"state": "empty"} -SECTION_TO_REPLACE = "__section_to_replace" -ERR_SECTION_MISSING_NAME = "Error: section [{0}] item is missing 'name': {1}" -ERR_NAME_NOT_VALID = "Error: section [{0}] item name [{1}] is not a valid string" -ERR_NO_VALUE_OR_STATE = ( - "Error: section [{0}] item name [{1}] must have either a 'value' or 'state'" -) -ERR_BOTH_VALUE_AND_STATE = ( - "Error: section [{0}] item name [{1}] must have only one of 'value' or 'state'" -) -ERR_STATE_ABSENT = ( - "Error: section [{0}] item name [{1}] state value must be 'absent' not [{2}]" -) -ERR_UNEXPECTED_VALUES = "Error: section [{0}] item [{1}] has unexpected values {2}" -ERR_VALUE_ITEM_NOT_DICT = ( - "Error: section [{0}] item name [{1}] value [{2}] is not a dict" -) -ERR_VALUE_ITEM_PREVIOUS = ( - "Error: section [{0}] item name [{1}] has invalid value for 'previous' [{2}]" -) -ERR_REMOVE_SECTION_VALUE = "Error: to remove the section [{0}] specify the value {1}" -ERR_ITEM_NOT_DICT = "Error: section [{0}] item value [{1}] is not a dict" -ERR_ITEM_PREVIOUS = "Error: section [{0}] item has invalid value for 'previous' [{1}]" -ERR_ITEM_DICT_OR_LIST = "Error: section [{0}] value must be a dict or a list" -ERR_LIST_NOT_ALLOWED = "Error: section [{0}] item [{1}] has unexpected list value {2}" -ERR_BLCMD_MUST_BE_LIST = "Error: section [{0}] item [{1}] must be a list not [{2}]" -ERR_VALUE_CANNOT_BE_BOOLEAN = ( - "Error: section [{0}] item [{1}] value [{2}] must not " - "be a boolean - try quoting the value" -) - - -def get_supported_tuned_plugin_names(): - """get names of all tuned plugins supported by this module""" - return [ - "bootloader", - "modules", - "selinux", - "sysctl", - "sysfs", - "systemd", - "vm", - "cpu", - "disk", - "net", - "audio", - "scsi_host", - "video", - "usb", - ] - - -def no_such_profile(): - """see if the last log message was that the profile did not exist""" - lastlogmsg = tuned.logs.root_logger.logstack[-1].msg - return re.search("Requested profile .* doesn't exist", lastlogmsg) is not None - - -def profile_to_dict(profile): - """convert profile object to dict""" - ret_val = {} - for unitname, unit in profile.units.items(): - ret_val[unitname] = unit.options - return ret_val - - -def debug_print_profile(profile, amodule): - """for debugging - print profile as INI""" - amodule.debug("profile {0}".format(profile.name)) - amodule.debug(str(profile_to_dict(profile))) - - -caught_name_error = None -try: - EMPTYUNIT = tuned.profiles.unit.Unit("empty", {}) -except NameError as nerr: - # tuned not loaded in check mode - caught_name_error = nerr - - -def get_profile_unit_key(profile, unitname, key): - """convenience function""" - return profile.units.get(unitname, EMPTYUNIT).options.get(key) - - -class BLCmdLine(object): - """A data type for handling bootloader cmdline values.""" - - def __init__(self, val): - self.key_list = [] # list of keys in order - to preserve ordering - self.key_to_val = {} # maps key to value - if val: - for item in shlex.split(val): - key, val = self.splititem(item) - self.key_list.append(key) - # None or '' means bare key - no value - self.key_to_val[key] = val - - @classmethod - def splititem(cls, item): - """split item in form key=somevalue into key and somevalue""" - # pylint: disable=blacklisted-name - key, _, val = item.partition("=") - return key, val - - @classmethod - def escapeval(cls, val): - """make sure val is quoted as in shell""" - return ansible_six_moves.shlex_quote(str(val)) - - def __str__(self): - vallist = [] - for key in self.key_list: - val = self.key_to_val[key] - if val: - vallist.append("{0}={1}".format(key, self.escapeval(val))) - else: - vallist.append(key) - return " ".join(vallist) - - def add(self, key, val): - """add/replace the given key & value""" - if key not in self.key_to_val: - self.key_list.append(key) - self.key_to_val[key] = val - - def remove(self, key): - """remove the given key""" - if key in self.key_to_val: - self.key_list.remove(key) - del self.key_to_val[key] - - -def apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue): - """apply a bootloader cmdline item to the current profile""" - name = item["name"] - if item.get(SECTION_TO_REPLACE, False): - blcmd = BLCmdLine("") - else: - blcmd = BLCmdLine(curvalue) - for subitem in item["value"]: - if "previous" in subitem: - continue - if subitem.get("state") == "absent": - blcmd.remove(subitem["name"]) - else: - blcmd.add(subitem["name"], subitem.get("value")) - blcmdstr = str(blcmd) - if blcmdstr: - current_profile.units.setdefault( - unitname, tuned.profiles.unit.Unit(unitname, {}) - ).options[name] = blcmdstr - elif curvalue: - del current_profile.units[unitname].options[name] - - -def apply_item_to_profile(item, unitname, current_profile): - """apply a specific item from a section to the current_profile""" - name = item["name"] - curvalue = get_profile_unit_key(current_profile, unitname, name) - newvalue = item.get("value") - if item.get("state", None) == "absent": - if curvalue: - del current_profile.units[unitname].options[name] - elif unitname == "bootloader" and name == "cmdline": - apply_bootloader_cmdline_item(item, unitname, current_profile, curvalue) - else: - current_profile.units.setdefault( - unitname, tuned.profiles.unit.Unit(unitname, {}) - ).options[name] = str(newvalue) - - -def is_reboot_required(unitname): - """Some changes need a reboot for the changes to be applied - For example, bootloader cmdline changes""" - # for now, only bootloader cmdline changes need a reboot - return unitname == "bootloader" - - -def apply_params_to_profile(params, current_profile, purge): - """apply the settings from the input parameters to the current profile - delete the unit if it is empty after applying the parameter deletions - """ - changestatus = NOCHANGES - reboot_required = False - section_to_replace = params.pop(SECTION_TO_REPLACE, {}) - need_purge = set() - if purge: - # remove units not specified in params - need_purge = set(current_profile.units.keys()) - for unitname, items in params.items(): - unit = current_profile.units.get(unitname, None) - current_options = {} - if unit: - current_options = copy.deepcopy(unit.options) - replace = section_to_replace.get(unitname, purge) - if replace or (items == REMOVE_SECTION_VALUE): - if unit: - unit.options.clear() - if purge and unitname in need_purge: - need_purge.remove(unitname) - if items == REMOVE_SECTION_VALUE: - if unit: - # we changed the profile - changestatus = CHANGES - reboot_required = reboot_required or is_reboot_required(unitname) - # we're done - no further processing necessary for this unit - continue - for item in items: - if item and "previous" not in item: - apply_item_to_profile(item, unitname, current_profile) - newoptions = {} - if unitname in current_profile.units: - newoptions = current_profile.units[unitname].options - if current_options != newoptions: - changestatus = CHANGES - reboot_required = reboot_required or is_reboot_required(unitname) - for unitname in need_purge: - del current_profile.units[unitname] - changestatus = CHANGES - reboot_required = reboot_required or is_reboot_required(unitname) - # remove empty units - for unitname in list(current_profile.units.keys()): - if not current_profile.units[unitname].options: - del current_profile.units[unitname] - return changestatus, reboot_required - - -def write_profile(current_profile): - """write the profile to the profile file""" - # convert profile to configobj to write ini-style file - # profile.options go into [main] section - # profile.units go into [unitname] section - cfg = configobj.ConfigObj(indent_type="") - cfg.initial_comment = ["File managed by Ansible - DO NOT EDIT"] - cfg["main"] = current_profile.options - for unitname, unit in current_profile.units.items(): - cfg[unitname] = unit.options - profile_base_dir = tuned.consts.LOAD_DIRECTORIES[-1] - prof_fname = os.path.join( - profile_base_dir, TUNED_PROFILE, tuned.consts.PROFILE_FILE - ) - with open(prof_fname, "wb") as prof_f: - cfg.write(prof_f) - - -def update_current_profile_and_mode(daemon, profile_list): - """ensure that the tuned current_profile applies the kernel_settings last""" - changed = False - profile, manual = daemon._get_startup_profile() - # is TUNED_PROFILE in the list? - profile_list.extend(profile.split()) - if TUNED_PROFILE not in profile_list: - changed = True - profile_list.append(TUNED_PROFILE) - # have to convert to manual mode in order to ensure kernel_settings are applied - if not manual: - changed = True - manual = True - if changed: - daemon._save_active_profile(" ".join(profile_list), manual) - return changed - - -def setup_for_testing(): - """create an /etc/tuned and /usr/lib/tuned directory structure for testing""" - test_root_dir = os.environ.get("TEST_ROOT_DIR") - if test_root_dir is None: - test_root_dir = tempfile.mkdtemp(suffix=".lsr") - # copy all of the test configs and profiles - test_root_dir_tuned = os.path.join(test_root_dir, "tuned") - test_src_dir = os.environ.get("TEST_SRC_DIR", "tests") - src_dir = os.path.join(test_src_dir, "tuned") - shutil.copytree(src_dir, test_root_dir_tuned) - # patch all of the consts to use the test_root_dir - orig_consts = {} - for cnst in ( - "GLOBAL_CONFIG_FILE", - "ACTIVE_PROFILE_FILE", - "PROFILE_MODE_FILE", - "RECOMMEND_CONF_FILE", - "BOOT_CMDLINE_FILE", - ): - orig_consts[cnst] = tuned.consts.__dict__[cnst] - fname = os.path.join( - test_root_dir_tuned, - os.path.relpath(tuned.consts.__dict__[cnst], os.path.sep), - ) - tuned.consts.__dict__[cnst] = fname - dname = os.path.dirname(fname) - if not os.path.isdir(dname): - os.makedirs(dname) - orig_load_dirs = [] - for idx, dname in enumerate(tuned.consts.LOAD_DIRECTORIES): - orig_load_dirs.append(dname) - newdname = os.path.join( - test_root_dir_tuned, os.path.relpath(dname, os.path.sep) - ) - tuned.consts.LOAD_DIRECTORIES[idx] = newdname - if not os.path.isdir(newdname): - os.makedirs(newdname) - orig_rec_dirs = [] - for idx, dname in enumerate(tuned.consts.RECOMMEND_DIRECTORIES): - orig_rec_dirs.append(dname) - newdname = os.path.join( - test_root_dir_tuned, os.path.relpath(dname, os.path.sep) - ) - tuned.consts.RECOMMEND_DIRECTORIES[idx] = newdname - if not os.path.isdir(newdname): - os.makedirs(newdname) - has_func = bool( - getattr(tuned.utils.global_config.GlobalConfig.__init__, "__func__", False) - ) - if has_func: - orig_gc_init_defaults = ( - tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ - ) - orig_gc_load_config_defaults = ( - tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ - ) - if orig_gc_init_defaults: - tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - if orig_gc_load_config_defaults: - tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - else: - orig_gc_init_defaults = ( - tuned.utils.global_config.GlobalConfig.__init__.__defaults__ - ) - orig_gc_load_config_defaults = ( - tuned.utils.global_config.GlobalConfig.load_config.__defaults__ - ) - if orig_gc_init_defaults: - tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - if orig_gc_load_config_defaults: - tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( - tuned.consts.GLOBAL_CONFIG_FILE, - ) - # this call fails on ubuntu and containers, so mock it for testing - import pyudev.monitor - - orig_set_receive_buffer_size = pyudev.monitor.Monitor.set_receive_buffer_size - pyudev.monitor.Monitor.set_receive_buffer_size = lambda self, size: None - - def test_cleanup(): - import os - import shutil - - if "TEST_ROOT_DIR" not in os.environ: - shutil.rmtree(test_root_dir) - for cnst, val in orig_consts.items(): - tuned.consts.__dict__[cnst] = val - for idx, dname in enumerate(orig_load_dirs): - tuned.consts.LOAD_DIRECTORIES[idx] = dname - for idx, dname in enumerate(orig_rec_dirs): - tuned.consts.RECOMMEND_DIRECTORIES[idx] = dname - if has_func: - tuned.utils.global_config.GlobalConfig.__init__.__func__.__defaults__ = ( - orig_gc_init_defaults - ) - tuned.utils.global_config.GlobalConfig.load_config.__func__.__defaults__ = ( - orig_gc_load_config_defaults - ) - else: - tuned.utils.global_config.GlobalConfig.__init__.__defaults__ = ( - orig_gc_init_defaults - ) - tuned.utils.global_config.GlobalConfig.load_config.__defaults__ = ( - orig_gc_load_config_defaults - ) - pyudev.monitor.Monitor.set_receive_buffer_size = orig_set_receive_buffer_size - - if "TEST_ROOT_DIR" not in os.environ: - atexit.register(test_cleanup) - return test_cleanup - - -def get_tuned_config(): - """get the tuned config and set our parameters in it""" - tuned_config = tuned.utils.global_config.GlobalConfig() - tuned_config.set("daemon", 0) - tuned_config.set("reapply_sysctl", 0) - tuned_config.set("dynamic_tuning", 0) - return tuned_config - - -def load_current_profile(tuned_config, tuned_profile, logger): - """load the current profile""" - tuned_app = None - errmsg = "Error loading tuned profile [{0}]".format(TUNED_PROFILE) - try: - tuned_app = tuned.daemon.Application(tuned_profile, tuned_config) - except TunedException as tex: - logger.debug("caught TunedException [{0}]".format(tex)) - errmsg = errmsg + ": {0}".format(tex) - tuned_app = None - except IOError as ioe: - # for testing this case, need to create a profile with a bad permissions e.g. - # mkdir ioerror_profile; touch ioerror_profile/tuned.conf; chmod 0000 !$ - logger.debug("caught IOError [{0}]".format(ioe)) - errmsg = errmsg + ": {0}".format(ioe) - tuned_app = None - if ( - tuned_app is None - or tuned_app.daemon is None - or tuned_app.daemon.profile is None - or tuned_app.daemon.profile.units is None - or tuned_app.daemon.profile.options is None - or "summary" not in tuned_app.daemon.profile.options - ): - tuned_app = None - if no_such_profile(): - errmsg = errmsg + ": Profile does not exist" - return tuned_app, errmsg - - -def validate_and_digest_item(sectionname, item, listallowed=True, allowempty=False): - """Validate an item - must contain only name, value, and state""" - tmpitem = item.copy() - name = tmpitem.pop("name", None) - value = tmpitem.pop("value", None) - state = tmpitem.pop("state", None) - errlist = [] - if name is None: - errlist.append(ERR_SECTION_MISSING_NAME.format(sectionname, item)) - elif not isinstance(name, ansible_six.string_types): - errlist.append(ERR_NAME_NOT_VALID.format(sectionname, name)) - elif (value is None and not allowempty) and state is None: - errlist.append(ERR_NO_VALUE_OR_STATE.format(sectionname, name)) - elif value is not None and state is not None: - errlist.append(ERR_BOTH_VALUE_AND_STATE.format(sectionname, name)) - elif state is not None and state != "absent": - errlist.append(ERR_STATE_ABSENT.format(sectionname, name, state)) - elif tmpitem: - errlist.append(ERR_UNEXPECTED_VALUES.format(sectionname, name, tmpitem)) - elif isinstance(value, list): - if not listallowed: - errlist.append(ERR_LIST_NOT_ALLOWED.format(sectionname, name, value)) - else: - for valitem in value: - if not isinstance(valitem, dict): - errlist.append( - ERR_VALUE_ITEM_NOT_DICT.format(sectionname, name, valitem) - ) - elif "previous" in valitem: - if valitem["previous"] != "replaced": - errlist.append( - ERR_VALUE_ITEM_PREVIOUS.format( - sectionname, name, valitem["previous"] - ) - ) - else: - item[SECTION_TO_REPLACE] = True - else: - tmperrlist = validate_and_digest_item( - sectionname, valitem, False, True - ) - errlist.extend(tmperrlist) - elif sectionname == "bootloader" and name == "cmdline": - errlist.append(ERR_BLCMD_MUST_BE_LIST.format(sectionname, name, value)) - elif isinstance(value, bool): - errlist.append(ERR_VALUE_CANNOT_BE_BOOLEAN.format(sectionname, name, value)) - return errlist - - -def validate_and_digest(params): - """Validate that params is in the correct format, since we - are using type `raw`, we have to perform the validation here. - Also do some pre-processing to make it easier to apply - the params to profile""" - errlist = [] - replaces = {} - for sectionname, items in params.items(): - if isinstance(items, dict): - if not items == REMOVE_SECTION_VALUE: - errlist.append( - ERR_REMOVE_SECTION_VALUE.format(sectionname, REMOVE_SECTION_VALUE) - ) - elif isinstance(items, list): - for item in items: - if not isinstance(item, dict): - errlist.append(ERR_ITEM_NOT_DICT.format(sectionname, item)) - elif not item: - continue # ignore empty items - elif "previous" in item: - if item["previous"] != "replaced": - errlist.append( - ERR_ITEM_PREVIOUS.format(sectionname, item["previous"]) - ) - else: - replaces[sectionname] = True - else: - itemerrlist = validate_and_digest_item(sectionname, item) - errlist.extend(itemerrlist) - else: - errlist.append(ERR_ITEM_DICT_OR_LIST.format(sectionname)) - if replaces: - params[SECTION_TO_REPLACE] = replaces - - return errlist - - -def remove_if_empty(params): - """recursively remove empty items from params - return true if params results in being empty, - false otherwise""" - if isinstance(params, list): - removed = 0 - for idx in range(0, len(params)): - realidx = idx - removed - if remove_if_empty(params[realidx]): - del params[realidx] - removed = removed + 1 - elif isinstance(params, dict): - for key, val in list(params.items()): - if remove_if_empty(val): - del params[key] - return params == [] or params == {} or params == "" or params is None - - -def run_module(): - """ The entry point of the module. """ - - module_args = dict( - purge=dict(type="bool", required=False, default=False), - ) - tuned_plugin_names = get_supported_tuned_plugin_names() - for plugin_name in tuned_plugin_names: - # use raw here - type can be dict or list - perform validation - # below - module_args[plugin_name] = dict(type="raw", required=False) - - result = dict(changed=False, message="") - - module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) - - if not module.check_mode: - if os.environ.get("TESTING", "false") == "true": - # pylint: disable=blacklisted-name - _ = setup_for_testing() - - params = module.params - # remove any non-tuned fields from params and save them locally - # state = params.pop("state") - purge = params.pop("purge", False) - # also remove any empty or None - # pylint: disable=blacklisted-name - _ = remove_if_empty(params) - errlist = validate_and_digest(params) - if errlist: - errmsg = "Invalid format for input parameters" - module.fail_json(msg=errmsg, warnings=errlist, **result) - - # In check_mode, just perform input validation (above), because - # tuned will not be installed on the remote system - if module.check_mode: - module.exit_json(**result) - elif caught_import_error is not None: - raise caught_import_error # pylint: disable-msg=E0702 - elif caught_name_error is not None: - # name error is usually because tuned module was not imported - # but just in case, report it here - raise caught_name_error # pylint: disable-msg=E0702 - - tuned_config = get_tuned_config() - current_profile = None - tuned_app, errmsg = load_current_profile(tuned_config, TUNED_PROFILE, module) - - if tuned_app is None: - module.fail_json(msg=errmsg, **result) - else: - current_profile = tuned_app.daemon.profile - debug_print_profile(current_profile, module) - errmsg = "" - - result["msg"] = "Kernel settings were updated." - - # apply the given params to the profile - if there are any new items - # the function will return True we set changed = True - changestatus, reboot_required = apply_params_to_profile( - params, current_profile, purge - ) - profile_list = [] - if update_current_profile_and_mode(tuned_app.daemon, profile_list): - # profile or mode changed - if changestatus == NOCHANGES: - changestatus = CHANGES - result["msg"] = "Updated active profile and/or mode." - if changestatus > NOCHANGES: - try: - write_profile(current_profile) - # notify tuned to reload/reapply profile - except TunedException as tex: - module.debug("caught TunedException [{0}]".format(tex)) - errmsg = "Unable to apply tuned settings: {0}".format(tex) - module.fail_json(msg=errmsg, **result) - except IOError as ioe: - module.debug("caught IOError [{0}]".format(ioe)) - errmsg = "Unable to apply tuned settings: {0}".format(ioe) - module.fail_json(msg=errmsg, **result) - result["changed"] = True - else: - result["msg"] = "Kernel settings are up to date." - debug_print_profile(current_profile, module) - result["new_profile"] = profile_to_dict(current_profile) - result["active_profile"] = " ".join(profile_list) - result["reboot_required"] = reboot_required - if reboot_required: - result["msg"] = ( - result["msg"] + " A system reboot is needed to apply the changes." - ) - module.exit_json(**result) - - -def main(): - """ The main function! """ - run_module() - - -if __name__ == "__main__": - main() From 3fc17d61cb6012789920e8aad3da6aff0be8252c Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 8 Apr 2021 09:55:50 -0400 Subject: [PATCH 13/29] Add a couple more sysfs fields, fix bugs --- library/kernel_report.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/library/kernel_report.py b/library/kernel_report.py index 3be4d1d5..63aa7d68 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -50,7 +50,7 @@ def get_sysfs_fields(): for block in blocks: num_blocks += 1 if 'kernel_settings_disk_elevator' not in result: - result["kernel_settings_disk_elevator"] = re.findall(r'\[(\w+)\]', safe_file_get_contents("%s/queue/scheduler" % block.sys_path)) + result["kernel_settings_disk_elevator"] = re.findall(r'\[(\w+)\]', safe_file_get_contents("%s/queue/scheduler" % block.sys_path))[0] result["kernel_settings_disk_read_ahead_kb"] = int(safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)) result["kernel_settings_disk_scheduler_quantum"] = safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path) @@ -107,10 +107,21 @@ def get_sysfs_fields(): for usb in usbs: num_usbs += 1 if "kernel_settings_usb_autosuspend" not in result: - result["kernel_settings_usb_autosuspend"] = safe_file_get_contents("%s/power/autosuspend" % usb.sys_path) + result["kernel_settings_usb_autosuspend"] = int(safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)) print("get_sysfs_fields: found %d usbs associated to the template machine" % num_usbs) + other_sysfs = [] + ksm_dict = {"name": "/sys/kernel/mm/ksm/run"} + ksm_dict["value"] = safe_file_get_contents("/sys/kernel/mm/ksm/run") + other_sysfs.append(ksm_dict) + + ktimer_dict = {"name": "/sys/kernel/ktimer_lockless_check"} + ktimer_dict["value"] = safe_file_get_contents("/sys/kernel/ktimer_lockless_check") + other_sysfs.append(ktimer_dict) + + result["kernel_settings_sysfs"] = other_sysfs + return result def run_module(): From 4781a50942396011835b69f74054b985d27f1737 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 8 Apr 2021 10:17:49 -0400 Subject: [PATCH 14/29] Add sysctl fields to new kernel_report plugin --- library/kernel_report.py | 62 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/library/kernel_report.py b/library/kernel_report.py index 63aa7d68..1c3d957a 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -10,10 +10,54 @@ import pyudev from ansible.module_utils.basic import AnsibleModule -UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events', 'vm\.drop_caches'] -UNSTABLE_SYSFS_FIELDS = ['kernel\.debug', 'devices'] -SYSCTL_DIR = '/proc/sys' -SYSFS_DIR = '/sys' +SYSCTL_FIELDS = ['fs.aio-max-nr', + 'fs.file-max', + 'fs.inotify.max_user_watches', + 'kernel.hung_task_timeout_secs', + 'kernel.nmi_watchdog', + 'kernel.numa_balancing', + 'kernel.panic_on_oops', + 'kernel.pid_max', + 'kernel.printk', + 'kernel.sched_autogroup_enabled', + 'kernel.sched_latency_ns', + 'kernel.sched_migration_cost_ns', + 'kernel.sched_min_granularity_ns', + 'kernel.sched_rt_runtime_us', + 'kernel.sched_wakeup_granularity_ns', + 'kernel.sem', + 'kernel.shmall', + 'kernel.shmmax', + 'kernel.shmmni', + 'kernel.timer_migration', + 'net.core.busy_poll', + 'net.core.busy_read', + 'net.core.rmem_default', + 'net.core.rmem_max', + 'net.core.wmem_default', + 'net.core.wmem_max', + 'net.ipv4.ip_local_port_range', + 'net.ipv4.tcp_fastopen', + 'net.ipv4.tcp_rmem', + 'net.ipv4.tcp_timestamps', + 'net.ipv4.tcp_window_scaling', + 'net.ipv4.tcp_wmem', + 'net.ipv4.udp_mem', + 'net.netfilter.nf_conntrack_max', + 'vm.dirty_background_bytes', + 'vm.dirty_background_ratio', + 'vm.dirty_bytes', + 'vm.dirty_expire_centisecs', + 'vm.dirty_ratio', + 'vm.dirty_writeback_centisecs', + 'vm.hugepages_treat_as_movable', + 'vm.laptop_mode', + 'vm.max_map_count', + 'vm.min_free_kbytes', + 'vm.stat_interval', + 'vm.swappiness', + 'vm.zone_reclaim_mode'] + def safe_file_get_contents(filename): try: @@ -22,6 +66,14 @@ def safe_file_get_contents(filename): except FileNotFoundError as e: print('safe_file_get_contents: suppressed exception FileNotFoundError with content \'%s\'' % e) +def get_sysctl_fields(): + sysctl = [] + for setting in SYSCTL_FIELDS: + temp_dict = {"name": setting} + temp_dict["value"] = safe_file_get_contents("/proc/sys/%s" % setting.replace(".","/")) + sysctl.append(temp_dict) + return {"kernel_settings_sysctl": sysctl} + def get_sysfs_fields(): result = {} result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] @@ -140,7 +192,7 @@ def run_module(): if module.check_mode: module.exit_json(**result) - result['ansible_facts'] = get_sysfs_fields() + result['ansible_facts'] = {**get_sysfs_fields(), **get_sysctl_fields()} module.exit_json(**result) From d14a1f6f8ab22abd6f0b82791275c8261c9f825c Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 15 Apr 2021 09:31:52 -0400 Subject: [PATCH 15/29] Add handling for device specific settings Update apply_kernel_report_settings test --- library/kernel_report.py | 97 +++++++++++-------- templates/kernel_report_device_specific.j2 | 1 + tests/tests_apply_kernel_report_settings.yml | 43 ++++++-- ...tests_apply_kernel_report_settings_old.yml | 22 +++++ tests/vars/tests_CentOS_8.yml | 1 + tests/vars/tests_RedHat_8.yml | 1 + tests/vars/tests_default.yml | 1 + 7 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 templates/kernel_report_device_specific.j2 create mode 100644 tests/tests_apply_kernel_report_settings_old.yml diff --git a/library/kernel_report.py b/library/kernel_report.py index 1c3d957a..7989de9f 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -71,67 +71,82 @@ def get_sysctl_fields(): for setting in SYSCTL_FIELDS: temp_dict = {"name": setting} temp_dict["value"] = safe_file_get_contents("/proc/sys/%s" % setting.replace(".","/")) - sysctl.append(temp_dict) + if temp_dict["value"]: + sysctl.append(temp_dict) return {"kernel_settings_sysctl": sysctl} def get_sysfs_fields(): result = {} + + # these are the settings not specific to any particular device result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] result["kernel_settings_transparent_hugepages_defrag"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0] + result["kernel_settings_cpu_min_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct") + result["kernel_settings_cpu_max_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct") + result["kernel_settings_cpu_no_turbo"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo") + result["kernel_settings_avc_cache_threshold"] = safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold") + result["kernel_settings_nf_conntrack_hashsize"] = safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize") + + kernel_settings_device_specific = {} # will collect a list of cpus associated to the template machine, but only use settings from the first one cpus = pyudev.Context().list_devices(subsystem="cpu") num_cpus = 0 + kernel_settings_device_specific["kernel_settings_cpu_governor"] = [] + kernel_settings_device_specific["kernel_settings_sampling_down_factor"] = [] for cpu in cpus: num_cpus += 1 - if 'kernel_settings_cpu_governor' not in result: - result["kernel_settings_cpu_governor"] = safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path) - result["kernel_settings_sampling_down_factor"] = safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % result['kernel_settings_cpu_governor']) + kernel_settings_device_specific["kernel_settings_cpu_governor"].append({'device':cpu.sys_path, 'value':safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path)}) + kernel_settings_device_specific["kernel_settings_sampling_down_factor"].append({'device':cpu.sys_path, 'value':safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % kernel_settings_device_specific['kernel_settings_cpu_governor'][-1])}) print("get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus) - result["kernel_settings_cpu_min_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct") - result["kernel_settings_cpu_max_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct") - result["kernel_settings_cpu_no_turbo"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo") - # will collect a list of blocks associated to the template machine, but only use settings from the first one blocks = pyudev.Context().list_devices(subsystem="block") num_blocks = 0 + kernel_settings_device_specific["kernel_settings_disk_elevator"] =[] + kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"] = [] + kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"] = [] + for block in blocks: num_blocks += 1 - if 'kernel_settings_disk_elevator' not in result: - result["kernel_settings_disk_elevator"] = re.findall(r'\[(\w+)\]', safe_file_get_contents("%s/queue/scheduler" % block.sys_path))[0] - result["kernel_settings_disk_read_ahead_kb"] = int(safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)) - result["kernel_settings_disk_scheduler_quantum"] = safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path) + schedulers = safe_file_get_contents("%s/queue/scheduler" % block.sys_path) + if schedulers: + settings = re.findall(r'\[(\w+)\]', schedulers) + if settings: + kernel_settings_device_specific["kernel_settings_disk_elevator"].append({'device':block.sys_path, 'value':settings[0]}) + kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"].append({'device':block.sys_path, 'value':safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)}) + kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"].append({'device':block.sys_path, 'value':safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path)}) print("get_sysfs_fields: found %d blocks associated to the template machine" % num_blocks) - - result["kernel_settings_avc_cache_threshold"] = int(safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")) - result["kernel_settings_nf_conntrack_hashsize"] = int(safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize")) - # will collect a list of modules associated to the template machine, but only use settings from the first one - num_modules = 0 - devices = pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") + # will collect a list of sound cards associated to the template machine, but only use settings from the first one + num_sound_cards = 0 + sound_cards = pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") + + kernel_settings_device_specific["kernel_settings_audio_timeout"] = [] + kernel_settings_device_specific["kernel_settings_audio_reset_controller"] = [] - for device in devices: - module_name = device.parent.driver + for sound_card in sound_cards: + module_name = sound_card.parent.driver if module_name in ["snd_hda_intel", "snd_ac97_codec"]: - num_modules += 1 - result["kernel_settings_audio_timeout"] = safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name) - result["kernel_settings_audio_reset_controller"] = safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name) + num_sound_cards += 1 + kernel_settings_device_specific["kernel_settings_audio_timeout"].append({'device':sound_card.sys_path, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name)}) + kernel_settings_device_specific["kernel_settings_audio_reset_controller"].append({'device':sound_card.sys_path, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name)}) - print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_modules) + print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_sound_cards) # will collect a list of scsis associated to the template machine, but only use settings from the first one num_scsis = 0 scsis = pyudev.Context().list_devices(subsystem="scsi") + kernel_settings_device_specific["kernel_settings_scsi_host_alpm"] = [] + for scsi in scsis: num_scsis += 1 - if "kernel_settings_scsi_host_alpm" not in result: - result["kernel_settings_scsi_host_alpm"] = safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path) + kernel_settings_device_specific["kernel_settings_scsi_host_alpm"].append({'device':scsi.sys_path, 'value':safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path)}) print("get_sysfs_fields: found %d scsis associated to the template machine" % num_scsis) @@ -139,38 +154,44 @@ def get_sysfs_fields(): num_gcards = 0 gcards = pyudev.Context().list_devices(subsystem="drm").match_sys_name("card*").match_property("DEVTYPE", "drm_minor") + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"] = [] + for gcard in gcards: num_gcards += 1 method = safe_file_get_contents("%s/device/power_method" % gcard.sys_path) - if "kernel_settings_video_radeon_powersave" not in result: - if method == "profile": - result["kernel_settings_video_radeon_powersave"] = safe_file_get_contents("%s/device/power_profile" % gcard.sys_path) - elif method == "dynpm": - result["kernel_settings_video_radeon_powersave"] = "dynpm" - elif method == "dpm": - result["kernel_settings_video_radeon_powersave"] = "dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path) - + if method == "profile": + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_path, 'value':safe_file_get_contents("%s/device/power_profile" % gcard.sys_path)}) + elif method == "dynpm": + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_path, 'value':'dynpm'}) + elif method == "dpm": + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_path, 'value':"dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path)}) + print("get_sysfs_fields: found %d gcards associated to the template machine" % num_gcards) # will collect a list of usb interfaces associated to the template machine, but only use settings from the first one num_usbs = 0 usbs = pyudev.Context().list_devices(subsystem="usb") + kernel_settings_device_specific["kernel_settings_usb_autosuspend"] = [] + for usb in usbs: num_usbs += 1 - if "kernel_settings_usb_autosuspend" not in result: - result["kernel_settings_usb_autosuspend"] = int(safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)) + kernel_settings_device_specific["kernel_settings_usb_autosuspend"].append({'device':usb.sys_path,'value':safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)}) + + result["kernel_settings_device_specific"] = kernel_settings_device_specific print("get_sysfs_fields: found %d usbs associated to the template machine" % num_usbs) other_sysfs = [] ksm_dict = {"name": "/sys/kernel/mm/ksm/run"} ksm_dict["value"] = safe_file_get_contents("/sys/kernel/mm/ksm/run") - other_sysfs.append(ksm_dict) + if ksm_dict["value"]: + other_sysfs.append(ksm_dict) ktimer_dict = {"name": "/sys/kernel/ktimer_lockless_check"} ktimer_dict["value"] = safe_file_get_contents("/sys/kernel/ktimer_lockless_check") - other_sysfs.append(ktimer_dict) + if ktimer_dict["value"]: + other_sysfs.append(ktimer_dict) result["kernel_settings_sysfs"] = other_sysfs diff --git a/templates/kernel_report_device_specific.j2 b/templates/kernel_report_device_specific.j2 new file mode 100644 index 00000000..f9aa04e6 --- /dev/null +++ b/templates/kernel_report_device_specific.j2 @@ -0,0 +1 @@ +{{ hostvars[item].template_settings.ansible_facts.kernel_settings_device_specific }} \ No newline at end of file diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index cb40c566..f048a641 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -5,19 +5,50 @@ when: false tasks: + - name: Set version specific variables + include_vars: "{{ lookup('first_found', ffparams) }}" + vars: + ffparams: + files: + - "tests_{{ ansible_distribution }}_\ + {{ ansible_distribution_major_version }}.yml" + - "tests_default.yml" + paths: + - vars + + - name: Ensure pyudev package is installed + package: + name: "{{ __kernel_settings_test_pyudev_pkg }}" + state: present + - name: get and store config values from template machine kernel_report: register: template_settings - when: ansible_distribution == 'CentOS' - no_log: true + when: ansible_distribution == 'Fedora' + + - name: create file with device specific settings + template: + src: "/home/mprovenc/linux-system-roles/kernel_settings/templates/kernel_report_device_specific.j2" + dest: "./device_specific_settings.txt" + delegate_to: localhost + loop: "{{ groups['all'] }}" + when: + - hostvars[item].ansible_distribution == 'Fedora' - name: apply kernel_settings to target machine(s) include_role: name: linux-system-roles.kernel_settings vars: - kernel_settings_sysctl: >- - {{ hostvars[item].template_settings.ansible_facts.sysctl }} + kernel_settings_transparent_hugepages: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages }}" + kernel_settings_transparent_hugepages_defrag: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages_defrag }}" + kernel_settings_cpu_min_perf_pct: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_cpu_min_perf_pct }}" + kernel_settings_cpu_max_perf_pct: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_cpu_max_perf_pct }}" + kernel_settings_cpu_no_turbo: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_cpu_no_turbo }}" + kernel_settings_avc_cache_threshold: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_avc_cache_threshold }}" + kernel_settings_nf_conntrack_hashsize: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_nf_conntrack_hashsize }}" + kernel_settings_sysfs: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_sysfs }}" + kernel_settings_sysctl: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_sysctl }}" loop: "{{ groups['all'] }}" when: - - hostvars[item].ansible_distribution == 'CentOS' - - ansible_distribution == 'Fedora' + - hostvars[item].ansible_distribution == 'Fedora' + - ansible_distribution == 'CentOS' diff --git a/tests/tests_apply_kernel_report_settings_old.yml b/tests/tests_apply_kernel_report_settings_old.yml new file mode 100644 index 00000000..ab7107c0 --- /dev/null +++ b/tests/tests_apply_kernel_report_settings_old.yml @@ -0,0 +1,22 @@ +- hosts: all + + roles: + - role: linux-system-roles.kernel_settings + when: false + + tasks: + - name: get and store config values from template machine + kernel_report: + register: template_settings + when: ansible_distribution == 'CentOS' + + - name: apply kernel_settings to target machine(s) + include_role: + name: linux-system-roles.kernel_settings + vars: + kernel_settings_sysctl: >- + {{ hostvars[item].template_settings.ansible_facts.sysctl }} + loop: "{{ groups['all'] }}" + when: + - hostvars[item].ansible_distribution == 'CentOS' + - ansible_distribution == 'Fedora' diff --git a/tests/vars/tests_CentOS_8.yml b/tests/vars/tests_CentOS_8.yml index f6f5e9f4..374c3d9e 100644 --- a/tests/vars/tests_CentOS_8.yml +++ b/tests/vars/tests_CentOS_8.yml @@ -1,2 +1,3 @@ __kernel_settings_test_python_pkgs: ['python3', 'python3-configobj'] __kernel_settings_test_python_cmd: python3 +__kernel_settings_test_pyudev_pkg: 'python3-pyudev' \ No newline at end of file diff --git a/tests/vars/tests_RedHat_8.yml b/tests/vars/tests_RedHat_8.yml index f6f5e9f4..0edfe629 100644 --- a/tests/vars/tests_RedHat_8.yml +++ b/tests/vars/tests_RedHat_8.yml @@ -1,2 +1,3 @@ __kernel_settings_test_python_pkgs: ['python3', 'python3-configobj'] __kernel_settings_test_python_cmd: python3 +__kernel_settings_test_pyudev_pkg: 'python3-pyudev' diff --git a/tests/vars/tests_default.yml b/tests/vars/tests_default.yml index b6b6c160..d00527db 100644 --- a/tests/vars/tests_default.yml +++ b/tests/vars/tests_default.yml @@ -1,2 +1,3 @@ __kernel_settings_test_python_pkgs: ['python', 'python-configobj'] __kernel_settings_test_python_cmd: python +__kernel_settings_test_pyudev_pkg: 'python-pyudev' From f2090ab5296cf92aba4a74135bd1f19f2351811f Mon Sep 17 00:00:00 2001 From: mprovenc Date: Fri, 16 Apr 2021 13:21:37 -0400 Subject: [PATCH 16/29] Add special logic around dirty_bytes/ratio sysctls Add special logic so that if one of these values is read in as '0', the kernel_report plugin will ignore it and only populate one of the bytes/ratio fields --- library/kernel_report.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/library/kernel_report.py b/library/kernel_report.py index 7989de9f..b904eb9f 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -72,7 +72,11 @@ def get_sysctl_fields(): temp_dict = {"name": setting} temp_dict["value"] = safe_file_get_contents("/proc/sys/%s" % setting.replace(".","/")) if temp_dict["value"]: - sysctl.append(temp_dict) + if (setting == "vm.dirty_background_bytes" or setting == "vm.dirty_background_ratio" or setting == "vm.dirty_ratio" or setting == "vm.dirty_bytes") and temp_dict["value"] == "0": + continue + else: + sysctl.append(temp_dict) + return {"kernel_settings_sysctl": sysctl} def get_sysfs_fields(): From b4f247884ea1b47228d88f9f504e5512dc9a6985 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Fri, 16 Apr 2021 13:51:01 -0400 Subject: [PATCH 17/29] change device output to use device name, not path --- library/kernel_report.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/library/kernel_report.py b/library/kernel_report.py index b904eb9f..5339a651 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -101,8 +101,8 @@ def get_sysfs_fields(): kernel_settings_device_specific["kernel_settings_sampling_down_factor"] = [] for cpu in cpus: num_cpus += 1 - kernel_settings_device_specific["kernel_settings_cpu_governor"].append({'device':cpu.sys_path, 'value':safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path)}) - kernel_settings_device_specific["kernel_settings_sampling_down_factor"].append({'device':cpu.sys_path, 'value':safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % kernel_settings_device_specific['kernel_settings_cpu_governor'][-1])}) + kernel_settings_device_specific["kernel_settings_cpu_governor"].append({'device':cpu.sys_name, 'value':safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path)}) + kernel_settings_device_specific["kernel_settings_sampling_down_factor"].append({'device':cpu.sys_name, 'value':safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % kernel_settings_device_specific['kernel_settings_cpu_governor'][-1])}) print("get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus) @@ -120,9 +120,9 @@ def get_sysfs_fields(): if schedulers: settings = re.findall(r'\[(\w+)\]', schedulers) if settings: - kernel_settings_device_specific["kernel_settings_disk_elevator"].append({'device':block.sys_path, 'value':settings[0]}) - kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"].append({'device':block.sys_path, 'value':safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)}) - kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"].append({'device':block.sys_path, 'value':safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path)}) + kernel_settings_device_specific["kernel_settings_disk_elevator"].append({'device':block.sys_name, 'value':settings[0]}) + kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"].append({'device':block.sys_name, 'value':safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)}) + kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"].append({'device':block.sys_name, 'value':safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path)}) print("get_sysfs_fields: found %d blocks associated to the template machine" % num_blocks) @@ -137,8 +137,8 @@ def get_sysfs_fields(): module_name = sound_card.parent.driver if module_name in ["snd_hda_intel", "snd_ac97_codec"]: num_sound_cards += 1 - kernel_settings_device_specific["kernel_settings_audio_timeout"].append({'device':sound_card.sys_path, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name)}) - kernel_settings_device_specific["kernel_settings_audio_reset_controller"].append({'device':sound_card.sys_path, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name)}) + kernel_settings_device_specific["kernel_settings_audio_timeout"].append({'device':sound_card.sys_name, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name)}) + kernel_settings_device_specific["kernel_settings_audio_reset_controller"].append({'device':sound_card.sys_name, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name)}) print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_sound_cards) @@ -150,7 +150,7 @@ def get_sysfs_fields(): for scsi in scsis: num_scsis += 1 - kernel_settings_device_specific["kernel_settings_scsi_host_alpm"].append({'device':scsi.sys_path, 'value':safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path)}) + kernel_settings_device_specific["kernel_settings_scsi_host_alpm"].append({'device':scsi.sys_name, 'value':safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path)}) print("get_sysfs_fields: found %d scsis associated to the template machine" % num_scsis) @@ -164,11 +164,11 @@ def get_sysfs_fields(): num_gcards += 1 method = safe_file_get_contents("%s/device/power_method" % gcard.sys_path) if method == "profile": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_path, 'value':safe_file_get_contents("%s/device/power_profile" % gcard.sys_path)}) + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_name, 'value':safe_file_get_contents("%s/device/power_profile" % gcard.sys_path)}) elif method == "dynpm": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_path, 'value':'dynpm'}) + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_name, 'value':'dynpm'}) elif method == "dpm": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_path, 'value':"dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path)}) + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_name, 'value':"dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path)}) print("get_sysfs_fields: found %d gcards associated to the template machine" % num_gcards) @@ -180,7 +180,7 @@ def get_sysfs_fields(): for usb in usbs: num_usbs += 1 - kernel_settings_device_specific["kernel_settings_usb_autosuspend"].append({'device':usb.sys_path,'value':safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)}) + kernel_settings_device_specific["kernel_settings_usb_autosuspend"].append({'device':usb.sys_name,'value':safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)}) result["kernel_settings_device_specific"] = kernel_settings_device_specific From 0b7cd5fac89edcad78119a46eedf34acd51dff40 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Tue, 20 Apr 2021 11:28:27 -0400 Subject: [PATCH 18/29] Add device support to role --- defaults/main.yml | 37 ++---- library/kernel_report.py | 37 ++++-- tasks/main.yml | 113 ++----------------- tests/tests_apply_kernel_report_settings.yml | 12 +- tests/tests_change_settings.yml | 63 ++++++++--- 5 files changed, 97 insertions(+), 165 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 79f47841..c3ca756e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -38,36 +38,13 @@ kernel_settings_transparent_hugepages: null # value. The actual supported values may be different depending on your OS. kernel_settings_transparent_hugepages_defrag: null -# cpu settings -kernel_settings_cpu_governor: null -kernel_settings_sampling_down_factor: null -kernel_settings_cpu_min_perf_pct: null -kernel_settings_cpu_max_perf_pct: null -kernel_settings_no_turbo: null - -# disk settings -kernel_settings_disk_elevator: null -kernel_settings_disk_read_ahead_kb: null -kernel_settings_disk_scheduler_quantum: null - -# selinux settings -kernel_settings_avc_cache_threshold: null - -# net settings -kernel_settings_nf_conntrack_hashsize: null - -# audio settings -kernel_settings_audio_timeout: null -kernel_settings_audio_reset_controller: null - -# scsi_host settings -kernel_settings_scsi_host_alpm: null - -# video settings -kernel_settings_video_radeon_powersave: null - -# usb settings -kernel_settings_usb_autosuspend: null +kernel_settings_cpu: [] +kernel_settings_disk: [] +kernel_settings_net: [] +kernel_settings_audio: [] +kernel_settings_scsi_host: [] +kernel_settings_video: [] +kernel_settings_usb: [] # If purge is true, completely wipe out whatever the current settings # are and replace them with kernel_settings_parameters diff --git a/library/kernel_report.py b/library/kernel_report.py index 5339a651..5d673c56 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -82,17 +82,36 @@ def get_sysctl_fields(): def get_sysfs_fields(): result = {} - # these are the settings not specific to any particular device - result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] - result["kernel_settings_transparent_hugepages_defrag"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0] - result["kernel_settings_cpu_min_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct") - result["kernel_settings_cpu_max_perf_pct"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct") - result["kernel_settings_cpu_no_turbo"] = safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo") - result["kernel_settings_avc_cache_threshold"] = safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold") - result["kernel_settings_nf_conntrack_hashsize"] = safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize") - kernel_settings_device_specific = {} + # these are the settings not specific to any particular device + kernel_settings_other = {} + + kernel_settings_cpu = {"settings": []} + kernel_settings_net = {"settings": []} + kernel_settings_selinux = {"settings": []} + + if safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct"): + kernel_settings_cpu["settings"].append({'name': 'min_perf_pct', 'value': safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct")}) + if safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct"): + kernel_settings_cpu["settings"].append({'name': 'max_perf_pct', 'value': safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct")}) + if safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo"): + kernel_settings_cpu["settings"].append({'name': 'no_turbo', 'value': safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo")}) + if safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize"): + kernel_settings_net["settings"].append({'name': 'nf_conntrack_hashsize', 'value': safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize")}) + if safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold"): + kernel_settings_selinux["settings"].append({'name': 'avc_cache_threshold', 'value': safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")}) + + kernel_settings_other["kernel_settings_cpu"] = kernel_settings_cpu + kernel_settings_other["kernel_settings_net"] = kernel_settings_net + kernel_settings_other["kernel_settings_selinux"] = kernel_settings_selinux + + result["kernel_settings_other"] = kernel_settings_other + + # TODO + # result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] + # result["kernel_settings_transparent_hugepages_defrag"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0] + # will collect a list of cpus associated to the template machine, but only use settings from the first one cpus = pyudev.Context().list_devices(subsystem="cpu") num_cpus = 0 diff --git a/tasks/main.yml b/tasks/main.yml index 803ace3b..a4fbd402 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -41,111 +41,14 @@ - name: Apply kernel settings kernel_settings: - cpu: - - name: "{{ 'governor' if kernel_settings_cpu_governor - else none }}" - value: "{{ kernel_settings_cpu_governor - if kernel_settings_cpu_governor != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_cpu_governor == - __kernel_settings_state_absent else none }}" - - name: "{{ 'min_perf_pct' if kernel_settings_cpu_min_perf_pct - else none }}" - value: "{{ kernel_settings_cpu_min_perf_pct - if kernel_settings_cpu_min_perf_pct != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_cpu_min_perf_pct == - __kernel_settings_state_absent else none }}" - - name: "{{ 'max_perf_pct' if kernel_settings_cpu_max_perf_pct - else none }}" - value: "{{ kernel_settings_cpu_max_perf_pct - if kernel_settings_cpu_max_perf_pct != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_cpu_max_perf_pct == - __kernel_settings_state_absent else none }}" - - name: "{{ 'sampling_down_factor' if kernel_settings_sampling_down_factor - else none }}" - value: "{{ kernel_settings_sampling_down_factor - if kernel_settings_sampling_down_factor != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_sampling_down_factor == - __kernel_settings_state_absent else none }}" - - name: "{{ 'no_turbo' if kernel_settings_no_turbo - else none }}" - value: "{{ kernel_settings_no_turbo - if kernel_settings_no_turbo != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_no_turbo == - __kernel_settings_state_absent else none }}" - disk: - - name: "{{ 'elevator' if kernel_settings_disk_elevator - else none }}" - value: "{{ kernel_settings_disk_elevator - if kernel_settings_disk_elevator != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_disk_elevator == - __kernel_settings_state_absent else none }}" - - name: "{{ 'readahead' if kernel_settings_disk_read_ahead_kb - else none }}" - value: "{{ kernel_settings_disk_read_ahead_kb - if kernel_settings_disk_read_ahead_kb != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_disk_read_ahead_kb == - __kernel_settings_state_absent else none }}" - - name: "{{ 'scheduler_quantum' if kernel_settings_disk_scheduler_quantum - else none }}" - value: "{{ kernel_settings_disk_scheduler_quantum - if kernel_settings_disk_scheduler_quantum != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_disk_scheduler_quantum == - __kernel_settings_state_absent else none }}" - net: - - name: "{{ 'nf_conntrack_hashsize' if kernel_settings_nf_conntrack_hashsize - else none }}" - value: "{{ kernel_settings_nf_conntrack_hashsize - if kernel_settings_nf_conntrack_hashsize != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_nf_conntrack_hashsize == - __kernel_settings_state_absent else none }}" - audio: - - name: "{{ 'timeout' if kernel_settings_audio_timeout - else none }}" - value: "{{ kernel_settings_audio_timeout - if kernel_settings_audio_timeout != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_audio_timeout == - __kernel_settings_state_absent else none }}" - - name: "{{ 'reset_controller' if kernel_settings_audio_reset_controller - else none }}" - value: "{{ kernel_settings_audio_reset_controller - if kernel_settings_audio_reset_controller != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_audio_reset_controller == - __kernel_settings_state_absent else none }}" - scsi_host: - - name: "{{ 'alpm' if kernel_settings_scsi_host_alpm - else none }}" - value: "{{ kernel_settings_scsi_host_alpm - if kernel_settings_scsi_host_alpm != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_scsi_host_alpm == - __kernel_settings_state_absent else none }}" - video: - - name: "{{ 'radeon_powersave' if kernel_settings_video_radeon_powersave - else none }}" - value: "{{ kernel_settings_video_radeon_powersave - if kernel_settings_video_radeon_powersave != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_video_radeon_powersave == - __kernel_settings_state_absent else none }}" - usb: - - name: "{{ 'autosuspend' if kernel_settings_usb_autosuspend - else none }}" - value: "{{ kernel_settings_usb_autosuspend - if kernel_settings_usb_autosuspend != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_usb_autosuspend == - __kernel_settings_state_absent else none }}" + cpu: "{{ kernel_settings_cpu.settings | union([{'name': 'devices', 'value': kernel_settings_cpu.devices | d('*', true)}]) if kernel_settings_cpu else omit }}" + disk: "{{ kernel_settings_disk.settings | union([{'name': 'devices', 'value': kernel_settings_disk.devices | d('*', true)}]) if kernel_settings_disk else omit }}" + net: "{{ kernel_settings_net.settings | union([{'name': 'devices', 'value': kernel_settings_net.devices | d('*', true)}]) if kernel_settings_net else omit }}" + audio: "{{ kernel_settings_audio.settings | union([{'name': 'devices', 'value': kernel_settings_audio.devices | d('*', true)}]) if kernel_settings_audio else omit }}" + scsi_host: "{{ kernel_settings_scsi_host.settings | union([{'name': 'devices', 'value': kernel_settings_scsi_host.devices | d('*', true)}]) if kernel_settings_scsi_host else omit }}" + video: "{{ kernel_settings_video.settings | union([{'name': 'devices', 'value': kernel_settings_video.devices | d('*', true)}]) if kernel_settings_video else omit }}" + usb: "{{ kernel_settings_usb.settings | union([{'name': 'devices', 'value': kernel_settings_usb.devices | d('*', true)}]) if kernel_settings_usb else omit }}" + selinux: "{{ kernel_settings_selinux.settings | union([{'name': 'devices', 'value': kernel_settings_selinux.devices | d('*', true)}]) if kernel_settings_selinux else omit }}" sysctl: "{{ kernel_settings_sysctl if kernel_settings_sysctl else omit }}" sysfs: "{{ kernel_settings_sysfs if kernel_settings_sysfs else omit }}" systemd: diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index f048a641..83d2f2c7 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -39,13 +39,11 @@ include_role: name: linux-system-roles.kernel_settings vars: - kernel_settings_transparent_hugepages: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages }}" - kernel_settings_transparent_hugepages_defrag: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages_defrag }}" - kernel_settings_cpu_min_perf_pct: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_cpu_min_perf_pct }}" - kernel_settings_cpu_max_perf_pct: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_cpu_max_perf_pct }}" - kernel_settings_cpu_no_turbo: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_cpu_no_turbo }}" - kernel_settings_avc_cache_threshold: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_avc_cache_threshold }}" - kernel_settings_nf_conntrack_hashsize: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_nf_conntrack_hashsize }}" + # kernel_settings_transparent_hugepages: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages }}" + # kernel_settings_transparent_hugepages_defrag: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages_defrag }}" + kernel_settings_cpu: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_other.kernel_settings_cpu }}" + kernel_settings_net: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_other.kernel_settings_net }}" + kernel_settings_selinux: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_other.kernel_settings_selinux }}" kernel_settings_sysfs: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_sysfs }}" kernel_settings_sysctl: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_sysctl }}" loop: "{{ groups['all'] }}" diff --git a/tests/tests_change_settings.yml b/tests/tests_change_settings.yml index c742d305..b494d3ab 100644 --- a/tests/tests_change_settings.yml +++ b/tests/tests_change_settings.yml @@ -35,20 +35,55 @@ kernel_settings_sysfs: - name: /sys/class/net/lo/mtu value: 65000 - kernel_settings_cpu_governor: "conservative|powersave" - kernel_settings_cpu_min_perf_pct: 20 - kernel_settings_cpu_max_perf_pct: 99 - kernel_settings_sampling_down_factor: 98 - kernel_settings_no_turbo: 1 - kernel_settings_disk_elevator: "bfq" - kernel_settings_disk_read_ahead_kb: 256 - kernel_settings_disk_scheduler_quantum: 64 - kernel_settings_nf_conntrack_hashsize: 1048576 - kernel_settings_audio_timeout: 10 - kernel_settings_audio_reset_controller: 1 - kernel_settings_scsi_host_alpm: "min_power" - kernel_settings_video_radeon_powersave: "auto" - kernel_settings_usb_autosuspend: 1 + kernel_settings_cpu: + devices: + settings: + - name: governor + value: "conservative|powersave" + - name: min_perf_pct + value: 20 + - name: max_perf_pct + value: 99 + - name: sampling_down_factor + value: 98 + - name: no_turbo + value: 1 + kernel_settings_disk: + devices: + settings: + - name: elevator + value: bfq + - name: read_ahead_kb + value: 256 + - name: scheduler_quantum + value: 64 + kernel_settings_net: + devices: + settings: + - name: nf_conntrack_hashsize + value: 1048576 + kernel_settings_audio: + devices: + settings: + - name: timeout + value: 10 + - name: reset_controller + value: 1 + kernel_settings_scsi_host: + devices: + settings: + - name: alpm + value: "min_power" + kernel_settings_video: + devices: + settings: + - name: radeon_powersave + value: "auto" + kernel_settings_usb: + devices: + settings: + - name: autosuspend + value: 1 - name: check sysfs after role runs command: grep -x 65000 /sys/class/net/lo/mtu From 147c73605bd36d6006770da1eeba05f9adabc2b8 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Tue, 20 Apr 2021 14:10:49 -0400 Subject: [PATCH 19/29] Generalize apply_settings test to use any release Tested with two Fedora machines sharing an identical release --- templates/kernel_report_device_specific.j2 | 2 +- tests/tests_apply_kernel_report_settings.yml | 59 +++++++++++++++----- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/templates/kernel_report_device_specific.j2 b/templates/kernel_report_device_specific.j2 index f9aa04e6..9be3f203 100644 --- a/templates/kernel_report_device_specific.j2 +++ b/templates/kernel_report_device_specific.j2 @@ -1 +1 @@ -{{ hostvars[item].template_settings.ansible_facts.kernel_settings_device_specific }} \ No newline at end of file +{{ template_settings.ansible_facts.kernel_settings_device_specific }} \ No newline at end of file diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index 83d2f2c7..ee04f042 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -4,6 +4,11 @@ - role: linux-system-roles.kernel_settings when: false + vars: + names: + - "template" + - "target" + tasks: - name: Set version specific variables include_vars: "{{ lookup('first_found', ffparams) }}" @@ -16,24 +21,55 @@ paths: - vars - - name: Ensure pyudev package is installed + - name: designate each host as either template or target + set_fact: + host_role: "{{ names[my_idx] }}" + delegate_to: "{{ item }}" + delegate_facts: true + loop: "{{ groups['all'] }}" + loop_control: + index_var: my_idx + run_once: true + + - name: debug the host role + debug: + msg: "{{ host_role }}" + + - name: Ensure pyudev package is installed on template machine package: name: "{{ __kernel_settings_test_pyudev_pkg }}" state: present + when: host_role == 'template' + + - name: update sysctl values on the template machine + become: true + sysctl: + name: fs.file-max + value: 100000 + state: present + reload: yes - name: get and store config values from template machine kernel_report: register: template_settings - when: ansible_distribution == 'Fedora' + when: host_role == 'template' - name: create file with device specific settings template: src: "/home/mprovenc/linux-system-roles/kernel_settings/templates/kernel_report_device_specific.j2" dest: "./device_specific_settings.txt" delegate_to: localhost + when: host_role == 'template' + run_once: true + + - name: set template_hostname fact + set_fact: + template_hostname: "{{ inventory_hostname }}" loop: "{{ groups['all'] }}" - when: - - hostvars[item].ansible_distribution == 'Fedora' + when: host_role == 'template' + loop_control: + index_var: my_idx + run_once: true - name: apply kernel_settings to target machine(s) include_role: @@ -41,12 +77,9 @@ vars: # kernel_settings_transparent_hugepages: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages }}" # kernel_settings_transparent_hugepages_defrag: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages_defrag }}" - kernel_settings_cpu: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_other.kernel_settings_cpu }}" - kernel_settings_net: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_other.kernel_settings_net }}" - kernel_settings_selinux: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_other.kernel_settings_selinux }}" - kernel_settings_sysfs: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_sysfs }}" - kernel_settings_sysctl: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_sysctl }}" - loop: "{{ groups['all'] }}" - when: - - hostvars[item].ansible_distribution == 'Fedora' - - ansible_distribution == 'CentOS' + kernel_settings_cpu: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_cpu }}" + kernel_settings_net: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_net }}" + kernel_settings_selinux: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_selinux }}" + kernel_settings_sysfs: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_sysfs }}" + kernel_settings_sysctl: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_sysctl }}" + when: host_role == 'target' From c4d259d41078ca8cac384fde46379a9c3d0769ca Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 22 Apr 2021 09:32:27 -0400 Subject: [PATCH 20/29] add kernel_settings_selinux to defaults --- defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/defaults/main.yml b/defaults/main.yml index c3ca756e..ab7772aa 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -45,6 +45,7 @@ kernel_settings_audio: [] kernel_settings_scsi_host: [] kernel_settings_video: [] kernel_settings_usb: [] +kernel_settings_selinux: [] # If purge is true, completely wipe out whatever the current settings # are and replace them with kernel_settings_parameters From 40f71e6a648acdfdd0a0041e2f7650c207d103b7 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 22 Apr 2021 10:17:41 -0400 Subject: [PATCH 21/29] Add transparent_hugepage parameters to krnl report --- defaults/main.yml | 1 + library/kernel_report.py | 10 ++++++---- tasks/main.yml | 18 +----------------- tests/tests_apply_kernel_report_settings.yml | 3 +-- 4 files changed, 9 insertions(+), 23 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index ab7772aa..c27164f1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -46,6 +46,7 @@ kernel_settings_scsi_host: [] kernel_settings_video: [] kernel_settings_usb: [] kernel_settings_selinux: [] +kernel_settings_vm: [] # If purge is true, completely wipe out whatever the current settings # are and replace them with kernel_settings_parameters diff --git a/library/kernel_report.py b/library/kernel_report.py index 5d673c56..f3d9c48d 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -90,6 +90,7 @@ def get_sysfs_fields(): kernel_settings_cpu = {"settings": []} kernel_settings_net = {"settings": []} kernel_settings_selinux = {"settings": []} + kernel_settings_vm = {"settings": []} if safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct"): kernel_settings_cpu["settings"].append({'name': 'min_perf_pct', 'value': safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct")}) @@ -101,17 +102,18 @@ def get_sysfs_fields(): kernel_settings_net["settings"].append({'name': 'nf_conntrack_hashsize', 'value': safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize")}) if safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold"): kernel_settings_selinux["settings"].append({'name': 'avc_cache_threshold', 'value': safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")}) + if safe_file_get_contents("/sys/kernel/mm/transparent_hugepage/enabled"): + kernel_settings_vm["settings"].append({'name':'transparent_hugepage', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0]}) + if safe_file_get_contents("/sys/kernel/mm/transparent_hugepage/defrag"): + kernel_settings_vm["settings"].append({'name':'transparent_hugepage.defrag', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0]}) kernel_settings_other["kernel_settings_cpu"] = kernel_settings_cpu kernel_settings_other["kernel_settings_net"] = kernel_settings_net kernel_settings_other["kernel_settings_selinux"] = kernel_settings_selinux + kernel_settings_other["kernel_settings_vm"] = kernel_settings_vm result["kernel_settings_other"] = kernel_settings_other - # TODO - # result["kernel_settings_transparent_hugepages"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0] - # result["kernel_settings_transparent_hugepages_defrag"] = re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0] - # will collect a list of cpus associated to the template machine, but only use settings from the first one cpus = pyudev.Context().list_devices(subsystem="cpu") num_cpus = 0 diff --git a/tasks/main.yml b/tasks/main.yml index a4fbd402..e8a6e3fc 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -49,6 +49,7 @@ video: "{{ kernel_settings_video.settings | union([{'name': 'devices', 'value': kernel_settings_video.devices | d('*', true)}]) if kernel_settings_video else omit }}" usb: "{{ kernel_settings_usb.settings | union([{'name': 'devices', 'value': kernel_settings_usb.devices | d('*', true)}]) if kernel_settings_usb else omit }}" selinux: "{{ kernel_settings_selinux.settings | union([{'name': 'devices', 'value': kernel_settings_selinux.devices | d('*', true)}]) if kernel_settings_selinux else omit }}" + vm: "{{ kernel_settings_vm.settings | union([{'name': 'devices', 'value': kernel_settings_vm.devices | d('*', true)}]) if kernel_settings_vm else omit }}" sysctl: "{{ kernel_settings_sysctl if kernel_settings_sysctl else omit }}" sysfs: "{{ kernel_settings_sysfs if kernel_settings_sysfs else omit }}" systemd: @@ -60,23 +61,6 @@ else none }}" state: "{{ 'absent' if kernel_settings_systemd_cpu_affinity == __kernel_settings_state_absent else none }}" - vm: - - name: "{{ 'transparent_hugepages' - if kernel_settings_transparent_hugepages - else none }}" - value: "{{ kernel_settings_transparent_hugepages - if kernel_settings_transparent_hugepages != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_transparent_hugepages == - __kernel_settings_state_absent else none }}" - - name: "{{ 'transparent_hugepage.defrag' - if kernel_settings_transparent_hugepages_defrag - else none }}" - value: "{{ kernel_settings_transparent_hugepages_defrag - if kernel_settings_transparent_hugepages_defrag != - __kernel_settings_state_absent else none }}" - state: "{{ 'absent' if kernel_settings_transparent_hugepages_defrag == - __kernel_settings_state_absent else none }}" bootloader: - name: "{{ 'cmdline' if kernel_settings_bootloader_cmdline | d({}) diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index ee04f042..ea94043a 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -75,11 +75,10 @@ include_role: name: linux-system-roles.kernel_settings vars: - # kernel_settings_transparent_hugepages: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages }}" - # kernel_settings_transparent_hugepages_defrag: "{{ hostvars[item].template_settings.ansible_facts.kernel_settings_transparent_hugepages_defrag }}" kernel_settings_cpu: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_cpu }}" kernel_settings_net: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_net }}" kernel_settings_selinux: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_selinux }}" + kernel_settings_vm: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_vm }}" kernel_settings_sysfs: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_sysfs }}" kernel_settings_sysctl: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_sysctl }}" when: host_role == 'target' From 4fffbec85f59bba8fdadbe00961b8afdca133a57 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 22 Apr 2021 15:30:28 -0400 Subject: [PATCH 22/29] code cleanup, start of unit test for kernel_report --- library/kernel_report.py | 149 +++++++++--------- library/old_kernel_report.py | 67 -------- tasks/main.yml | 45 ++++-- tests/tests_apply_kernel_report_settings.yml | 41 +++-- ...tests_apply_kernel_report_settings_old.yml | 22 --- tests/unit/modules/test_kernel_report.py | 28 ++++ tests/vars/tests_CentOS_8.yml | 2 +- 7 files changed, 172 insertions(+), 182 deletions(-) delete mode 100644 library/old_kernel_report.py delete mode 100644 tests/tests_apply_kernel_report_settings_old.yml create mode 100644 tests/unit/modules/test_kernel_report.py diff --git a/library/kernel_report.py b/library/kernel_report.py index f3d9c48d..7712e0e8 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -10,67 +10,68 @@ import pyudev from ansible.module_utils.basic import AnsibleModule -SYSCTL_FIELDS = ['fs.aio-max-nr', - 'fs.file-max', - 'fs.inotify.max_user_watches', - 'kernel.hung_task_timeout_secs', - 'kernel.nmi_watchdog', - 'kernel.numa_balancing', - 'kernel.panic_on_oops', - 'kernel.pid_max', - 'kernel.printk', - 'kernel.sched_autogroup_enabled', - 'kernel.sched_latency_ns', - 'kernel.sched_migration_cost_ns', - 'kernel.sched_min_granularity_ns', - 'kernel.sched_rt_runtime_us', - 'kernel.sched_wakeup_granularity_ns', - 'kernel.sem', - 'kernel.shmall', - 'kernel.shmmax', - 'kernel.shmmni', - 'kernel.timer_migration', - 'net.core.busy_poll', - 'net.core.busy_read', - 'net.core.rmem_default', - 'net.core.rmem_max', - 'net.core.wmem_default', - 'net.core.wmem_max', - 'net.ipv4.ip_local_port_range', - 'net.ipv4.tcp_fastopen', - 'net.ipv4.tcp_rmem', - 'net.ipv4.tcp_timestamps', - 'net.ipv4.tcp_window_scaling', - 'net.ipv4.tcp_wmem', - 'net.ipv4.udp_mem', - 'net.netfilter.nf_conntrack_max', - 'vm.dirty_background_bytes', - 'vm.dirty_background_ratio', - 'vm.dirty_bytes', - 'vm.dirty_expire_centisecs', - 'vm.dirty_ratio', - 'vm.dirty_writeback_centisecs', - 'vm.hugepages_treat_as_movable', - 'vm.laptop_mode', - 'vm.max_map_count', - 'vm.min_free_kbytes', - 'vm.stat_interval', - 'vm.swappiness', - 'vm.zone_reclaim_mode'] +SYSCTL_FIELDS = ['fs.aio-max-nr', + 'fs.file-max', + 'fs.inotify.max_user_watches', + 'kernel.hung_task_timeout_secs', + 'kernel.nmi_watchdog', + 'kernel.numa_balancing', + 'kernel.panic_on_oops', + 'kernel.pid_max', + 'kernel.printk', + 'kernel.sched_autogroup_enabled', + 'kernel.sched_latency_ns', + 'kernel.sched_migration_cost_ns', + 'kernel.sched_min_granularity_ns', + 'kernel.sched_rt_runtime_us', + 'kernel.sched_wakeup_granularity_ns', + 'kernel.sem', + 'kernel.shmall', + 'kernel.shmmax', + 'kernel.shmmni', + 'kernel.timer_migration', + 'net.core.busy_poll', + 'net.core.busy_read', + 'net.core.rmem_default', + 'net.core.rmem_max', + 'net.core.wmem_default', + 'net.core.wmem_max', + 'net.ipv4.ip_local_port_range', + 'net.ipv4.tcp_fastopen', + 'net.ipv4.tcp_rmem', + 'net.ipv4.tcp_timestamps', + 'net.ipv4.tcp_window_scaling', + 'net.ipv4.tcp_wmem', + 'net.ipv4.udp_mem', + 'net.netfilter.nf_conntrack_max', + 'vm.dirty_background_bytes', + 'vm.dirty_background_ratio', + 'vm.dirty_bytes', + 'vm.dirty_expire_centisecs', + 'vm.dirty_ratio', + 'vm.dirty_writeback_centisecs', + 'vm.hugepages_treat_as_movable', + 'vm.laptop_mode', + 'vm.max_map_count', + 'vm.min_free_kbytes', + 'vm.stat_interval', + 'vm.swappiness', + 'vm.zone_reclaim_mode'] def safe_file_get_contents(filename): try: with open(filename) as f: return f.read().rstrip() - except FileNotFoundError as e: - print('safe_file_get_contents: suppressed exception FileNotFoundError with content \'%s\'' % e) + except IOError as e: + print('safe_file_get_contents: suppressed exception FileNotFoundError with content \'%s\'' % e) + def get_sysctl_fields(): sysctl = [] for setting in SYSCTL_FIELDS: temp_dict = {"name": setting} - temp_dict["value"] = safe_file_get_contents("/proc/sys/%s" % setting.replace(".","/")) + temp_dict["value"] = safe_file_get_contents("/proc/sys/%s" % setting.replace(".", "/")) if temp_dict["value"]: if (setting == "vm.dirty_background_bytes" or setting == "vm.dirty_background_ratio" or setting == "vm.dirty_ratio" or setting == "vm.dirty_bytes") and temp_dict["value"] == "0": continue @@ -79,6 +80,7 @@ def get_sysctl_fields(): return {"kernel_settings_sysctl": sysctl} + def get_sysfs_fields(): result = {} @@ -103,9 +105,9 @@ def get_sysfs_fields(): if safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold"): kernel_settings_selinux["settings"].append({'name': 'avc_cache_threshold', 'value': safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")}) if safe_file_get_contents("/sys/kernel/mm/transparent_hugepage/enabled"): - kernel_settings_vm["settings"].append({'name':'transparent_hugepage', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0]}) + kernel_settings_vm["settings"].append({'name': 'transparent_hugepage', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0]}) if safe_file_get_contents("/sys/kernel/mm/transparent_hugepage/defrag"): - kernel_settings_vm["settings"].append({'name':'transparent_hugepage.defrag', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0]}) + kernel_settings_vm["settings"].append({'name': 'transparent_hugepage.defrag', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0]}) kernel_settings_other["kernel_settings_cpu"] = kernel_settings_cpu kernel_settings_other["kernel_settings_net"] = kernel_settings_net @@ -117,21 +119,21 @@ def get_sysfs_fields(): # will collect a list of cpus associated to the template machine, but only use settings from the first one cpus = pyudev.Context().list_devices(subsystem="cpu") num_cpus = 0 - + kernel_settings_device_specific["kernel_settings_cpu_governor"] = [] kernel_settings_device_specific["kernel_settings_sampling_down_factor"] = [] for cpu in cpus: num_cpus += 1 - kernel_settings_device_specific["kernel_settings_cpu_governor"].append({'device':cpu.sys_name, 'value':safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path)}) - kernel_settings_device_specific["kernel_settings_sampling_down_factor"].append({'device':cpu.sys_name, 'value':safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % kernel_settings_device_specific['kernel_settings_cpu_governor'][-1])}) - + kernel_settings_device_specific["kernel_settings_cpu_governor"].append({'device': cpu.sys_name, 'value': safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path)}) + kernel_settings_device_specific["kernel_settings_sampling_down_factor"].append({'device': cpu.sys_name, 'value': safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % kernel_settings_device_specific['kernel_settings_cpu_governor'][-1])}) + print("get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus) # will collect a list of blocks associated to the template machine, but only use settings from the first one blocks = pyudev.Context().list_devices(subsystem="block") num_blocks = 0 - kernel_settings_device_specific["kernel_settings_disk_elevator"] =[] + kernel_settings_device_specific["kernel_settings_disk_elevator"] = [] kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"] = [] kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"] = [] @@ -141,12 +143,12 @@ def get_sysfs_fields(): if schedulers: settings = re.findall(r'\[(\w+)\]', schedulers) if settings: - kernel_settings_device_specific["kernel_settings_disk_elevator"].append({'device':block.sys_name, 'value':settings[0]}) - kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"].append({'device':block.sys_name, 'value':safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)}) - kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"].append({'device':block.sys_name, 'value':safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path)}) - + kernel_settings_device_specific["kernel_settings_disk_elevator"].append({'device': block.sys_name, 'value': settings[0]}) + kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"].append({'device': block.sys_name, 'value': safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)}) + kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"].append({'device': block.sys_name, 'value': safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path)}) + print("get_sysfs_fields: found %d blocks associated to the template machine" % num_blocks) - + # will collect a list of sound cards associated to the template machine, but only use settings from the first one num_sound_cards = 0 sound_cards = pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") @@ -158,8 +160,8 @@ def get_sysfs_fields(): module_name = sound_card.parent.driver if module_name in ["snd_hda_intel", "snd_ac97_codec"]: num_sound_cards += 1 - kernel_settings_device_specific["kernel_settings_audio_timeout"].append({'device':sound_card.sys_name, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name)}) - kernel_settings_device_specific["kernel_settings_audio_reset_controller"].append({'device':sound_card.sys_name, 'value':safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name)}) + kernel_settings_device_specific["kernel_settings_audio_timeout"].append({'device': sound_card.sys_name, 'value': safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name)}) + kernel_settings_device_specific["kernel_settings_audio_reset_controller"].append({'device': sound_card.sys_name, 'value': safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name)}) print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_sound_cards) @@ -171,7 +173,7 @@ def get_sysfs_fields(): for scsi in scsis: num_scsis += 1 - kernel_settings_device_specific["kernel_settings_scsi_host_alpm"].append({'device':scsi.sys_name, 'value':safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path)}) + kernel_settings_device_specific["kernel_settings_scsi_host_alpm"].append({'device': scsi.sys_name, 'value': safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path)}) print("get_sysfs_fields: found %d scsis associated to the template machine" % num_scsis) @@ -185,11 +187,11 @@ def get_sysfs_fields(): num_gcards += 1 method = safe_file_get_contents("%s/device/power_method" % gcard.sys_path) if method == "profile": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_name, 'value':safe_file_get_contents("%s/device/power_profile" % gcard.sys_path)}) + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device': gcard.sys_name, 'value': safe_file_get_contents("%s/device/power_profile" % gcard.sys_path)}) elif method == "dynpm": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_name, 'value':'dynpm'}) + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device': gcard.sys_name, 'value': 'dynpm'}) elif method == "dpm": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device':gcard.sys_name, 'value':"dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path)}) + kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device': gcard.sys_name, 'value': "dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path)}) print("get_sysfs_fields: found %d gcards associated to the template machine" % num_gcards) @@ -201,7 +203,7 @@ def get_sysfs_fields(): for usb in usbs: num_usbs += 1 - kernel_settings_device_specific["kernel_settings_usb_autosuspend"].append({'device':usb.sys_name,'value':safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)}) + kernel_settings_device_specific["kernel_settings_usb_autosuspend"].append({'device': usb.sys_name, 'value': safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)}) result["kernel_settings_device_specific"] = kernel_settings_device_specific @@ -222,6 +224,7 @@ def get_sysfs_fields(): return result + def run_module(): module_args = dict() @@ -238,12 +241,16 @@ def run_module(): if module.check_mode: module.exit_json(**result) - result['ansible_facts'] = {**get_sysfs_fields(), **get_sysctl_fields()} + # result['ansible_facts'] = {**get_sysfs_fields(), **get_sysctl_fields()} + result['ansible_facts'] = get_sysfs_fields() + result['ansible_facts'].update(get_sysctl_fields()) module.exit_json(**result) + def main(): run_module() + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/library/old_kernel_report.py b/library/old_kernel_report.py deleted file mode 100644 index 8182286a..00000000 --- a/library/old_kernel_report.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright: (c) 2021, Mary Provencher -# SPDX-License-Identifier: GPL-2.0-or-later -# -""" Generate kernel settings facts for a system """ - -import os -import re -import subprocess as sp -from ansible.module_utils.basic import AnsibleModule - -UNSTABLE_SYSCTL_FIELDS = ['kernel\.hostname', 'kernel\.domainname', 'dev', 'kernel\.ns_last_pid', 'net\.netfilter\.nf_conntrack_events', 'vm\.drop_caches'] -UNSTABLE_SYSFS_FIELDS = ['kernel\.debug', 'devices'] -SYSCTL_DIR = '/proc/sys' -SYSFS_DIR = '/sys' - -def file_get_contents(filename): - with open(filename) as f: - return f.read().rstrip() - -def settings_walk(dir, unstable): - result = [] - combined_unstable = "(" + ")|(".join(unstable) + ")" - for dirpath, dirs, files in os.walk(dir): - if files: - for file in files: - setting_path = dirpath + "/" + file - if(int(oct(os.stat(setting_path).st_mode)[-3:]) >= 644): - formatted_setting = str(setting_path[len(dir)+1:]).replace("/",".") - if re.match(combined_unstable,formatted_setting) is None: - try: - val = file_get_contents(setting_path) - if val: - result.append({'name': formatted_setting, "value": val}) - except OSError as e: - # read errors occur on some of the 'stable_secret' files - pass - return result - -def run_module(): - module_args = dict() - - result = dict( - changed=False, - ansible_facts=dict(), - ) - - module = AnsibleModule( - argument_spec=module_args, - supports_check_mode=True - ) - - if module.check_mode: - module.exit_json(**result) - - # result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS)} - result['ansible_facts'] = {"sysctl":settings_walk(SYSCTL_DIR, UNSTABLE_SYSCTL_FIELDS), "sysfs":settings_walk(SYSFS_DIR, UNSTABLE_SYSFS_FIELDS)} - - module.exit_json(**result) - -def main(): - run_module() - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/tasks/main.yml b/tasks/main.yml index e8a6e3fc..03adcdb7 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -41,15 +41,42 @@ - name: Apply kernel settings kernel_settings: - cpu: "{{ kernel_settings_cpu.settings | union([{'name': 'devices', 'value': kernel_settings_cpu.devices | d('*', true)}]) if kernel_settings_cpu else omit }}" - disk: "{{ kernel_settings_disk.settings | union([{'name': 'devices', 'value': kernel_settings_disk.devices | d('*', true)}]) if kernel_settings_disk else omit }}" - net: "{{ kernel_settings_net.settings | union([{'name': 'devices', 'value': kernel_settings_net.devices | d('*', true)}]) if kernel_settings_net else omit }}" - audio: "{{ kernel_settings_audio.settings | union([{'name': 'devices', 'value': kernel_settings_audio.devices | d('*', true)}]) if kernel_settings_audio else omit }}" - scsi_host: "{{ kernel_settings_scsi_host.settings | union([{'name': 'devices', 'value': kernel_settings_scsi_host.devices | d('*', true)}]) if kernel_settings_scsi_host else omit }}" - video: "{{ kernel_settings_video.settings | union([{'name': 'devices', 'value': kernel_settings_video.devices | d('*', true)}]) if kernel_settings_video else omit }}" - usb: "{{ kernel_settings_usb.settings | union([{'name': 'devices', 'value': kernel_settings_usb.devices | d('*', true)}]) if kernel_settings_usb else omit }}" - selinux: "{{ kernel_settings_selinux.settings | union([{'name': 'devices', 'value': kernel_settings_selinux.devices | d('*', true)}]) if kernel_settings_selinux else omit }}" - vm: "{{ kernel_settings_vm.settings | union([{'name': 'devices', 'value': kernel_settings_vm.devices | d('*', true)}]) if kernel_settings_vm else omit }}" + cpu: "{{ kernel_settings_cpu.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_cpu.devices | + d('*', true)}]) if kernel_settings_cpu else omit }}" + disk: "{{ kernel_settings_disk.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_disk.devices | + d('*', true)}]) if kernel_settings_disk else omit }}" + net: "{{ kernel_settings_net.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_net.devices | + d('*', true)}]) if kernel_settings_net else omit }}" + audio: "{{ kernel_settings_audio.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_audio.devices | + d('*', true)}]) if kernel_settings_audio else omit }}" + scsi_host: "{{ kernel_settings_scsi_host.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_scsi_host.devices | + d('*', true)}]) if kernel_settings_scsi_host else omit }}" + video: "{{ kernel_settings_video.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_video.devices | + d('*', true)}]) if kernel_settings_video else omit }}" + usb: "{{ kernel_settings_usb.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_usb.devices | + d('*', true)}]) if kernel_settings_usb else omit }}" + selinux: "{{ kernel_settings_selinux.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_selinux.devices | + d('*', true)}]) if kernel_settings_selinux else omit }}" + vm: "{{ kernel_settings_vm.settings | + union([{'name': 'devices',\ + 'value': kernel_settings_vm.devices | + d('*', true)}]) if kernel_settings_vm else omit }}" sysctl: "{{ kernel_settings_sysctl if kernel_settings_sysctl else omit }}" sysfs: "{{ kernel_settings_sysfs if kernel_settings_sysfs else omit }}" systemd: diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index ea94043a..93a0faf8 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -44,10 +44,10 @@ - name: update sysctl values on the template machine become: true sysctl: - name: fs.file-max - value: 100000 - state: present - reload: yes + name: fs.file-max + value: 100000 + state: present + reload: yes - name: get and store config values from template machine kernel_report: @@ -55,8 +55,9 @@ when: host_role == 'template' - name: create file with device specific settings - template: - src: "/home/mprovenc/linux-system-roles/kernel_settings/templates/kernel_report_device_specific.j2" + template: + src: "/home/mprovenc/linux-system-roles/kernel_settings/\ + templates/kernel_report_device_specific.j2" dest: "./device_specific_settings.txt" delegate_to: localhost when: host_role == 'template' @@ -75,10 +76,26 @@ include_role: name: linux-system-roles.kernel_settings vars: - kernel_settings_cpu: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_cpu }}" - kernel_settings_net: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_net }}" - kernel_settings_selinux: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_selinux }}" - kernel_settings_vm: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_other.kernel_settings_vm }}" - kernel_settings_sysfs: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_sysfs }}" - kernel_settings_sysctl: "{{ hostvars[template_hostname].template_settings.ansible_facts.kernel_settings_sysctl }}" + kernel_settings_cpu: "{{ hostvars[template_hostname].\ + template_settings.ansible_facts.\ + kernel_settings_other.\ + kernel_settings_cpu }}" + kernel_settings_net: "{{ hostvars[template_hostname].\ + template_settings.ansible_facts.\ + kernel_settings_other.\ + kernel_settings_net }}" + kernel_settings_selinux: "{{ hostvars[template_hostname].\ + template_settings.ansible_facts.\ + kernel_settings_other.\ + kernel_settings_selinux }}" + kernel_settings_vm: "{{ hostvars[template_hostname].\ + template_settings.ansible_facts.\ + kernel_settings_other.\ + kernel_settings_vm }}" + kernel_settings_sysfs: "{{ hostvars[template_hostname].\ + template_settings.ansible_facts.\ + kernel_settings_sysfs }}" + kernel_settings_sysctl: "{{ hostvars[template_hostname].\ + template_settings.ansible_facts.\ + kernel_settings_sysctl }}" when: host_role == 'target' diff --git a/tests/tests_apply_kernel_report_settings_old.yml b/tests/tests_apply_kernel_report_settings_old.yml deleted file mode 100644 index ab7107c0..00000000 --- a/tests/tests_apply_kernel_report_settings_old.yml +++ /dev/null @@ -1,22 +0,0 @@ -- hosts: all - - roles: - - role: linux-system-roles.kernel_settings - when: false - - tasks: - - name: get and store config values from template machine - kernel_report: - register: template_settings - when: ansible_distribution == 'CentOS' - - - name: apply kernel_settings to target machine(s) - include_role: - name: linux-system-roles.kernel_settings - vars: - kernel_settings_sysctl: >- - {{ hostvars[item].template_settings.ansible_facts.sysctl }} - loop: "{{ groups['all'] }}" - when: - - hostvars[item].ansible_distribution == 'CentOS' - - ansible_distribution == 'Fedora' diff --git a/tests/unit/modules/test_kernel_report.py b/tests/unit/modules/test_kernel_report.py new file mode 100644 index 00000000..da4bc6a7 --- /dev/null +++ b/tests/unit/modules/test_kernel_report.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2021, Mary Provencher +# SPDX-License-Identifier: GPL-2.0-or-later +# +""" Unit tests for kernel_report module """ + +import unittest + +try: + from unittest.mock import patch +except ImportError: + from mock import patch + +import kernel_report + + +class KernelReportGetFields(unittest.TestCase): + """test operations involving getting existing values from a template machine""" + + @patch('kernel_report.safe_file_get_contents') + def test_get_sysctl(self, mock): + """do various tests of get_sysctl_fields""" + kernel_report.get_sysctl_fields() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vars/tests_CentOS_8.yml b/tests/vars/tests_CentOS_8.yml index 374c3d9e..0edfe629 100644 --- a/tests/vars/tests_CentOS_8.yml +++ b/tests/vars/tests_CentOS_8.yml @@ -1,3 +1,3 @@ __kernel_settings_test_python_pkgs: ['python3', 'python3-configobj'] __kernel_settings_test_python_cmd: python3 -__kernel_settings_test_pyudev_pkg: 'python3-pyudev' \ No newline at end of file +__kernel_settings_test_pyudev_pkg: 'python3-pyudev' From 19ad7ace09f88692537ed0645b590252856eb5c1 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Mon, 26 Apr 2021 09:46:08 -0400 Subject: [PATCH 23/29] update some sysctl values in test --- tests/tests_apply_kernel_report_settings.yml | 5 +++-- tests/vars/tests_default.yml | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index 93a0faf8..c0409104 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -44,10 +44,11 @@ - name: update sysctl values on the template machine become: true sysctl: - name: fs.file-max - value: 100000 + name: "{{ item.name }}" + value: "{{ item.value }}" state: present reload: yes + loop: "{{ __kernel_settings_test_sysctl }}" - name: get and store config values from template machine kernel_report: diff --git a/tests/vars/tests_default.yml b/tests/vars/tests_default.yml index d00527db..2f7cc85f 100644 --- a/tests/vars/tests_default.yml +++ b/tests/vars/tests_default.yml @@ -1,3 +1,16 @@ __kernel_settings_test_python_pkgs: ['python', 'python-configobj'] __kernel_settings_test_python_cmd: python __kernel_settings_test_pyudev_pkg: 'python-pyudev' +__kernel_settings_test_sysctl: + - name: fs.file-max + value: '99999' + - name: fs.aio-max-nr + value: '70000' + - name: kernel.sched_migration_cost_ns + value: '5000000' + - name: kernel.sched_wakeup_granularity_ns + value: '900000' + - name: vm.dirty_expire_centisecs + value: '2000' + - name: kernel.panic_on_oops + value: '1' \ No newline at end of file From 1f672d9152489b3bd3239b30313978872432caa7 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Mon, 26 Apr 2021 12:26:16 -0400 Subject: [PATCH 24/29] add test for safe_file_get_contents --- tests/unit/modules/test_kernel_report.py | 25 ++++++++++++++++++++---- tests/vars/tests_default.yml | 2 +- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/unit/modules/test_kernel_report.py b/tests/unit/modules/test_kernel_report.py index da4bc6a7..67cb7b7c 100644 --- a/tests/unit/modules/test_kernel_report.py +++ b/tests/unit/modules/test_kernel_report.py @@ -5,12 +5,12 @@ # """ Unit tests for kernel_report module """ -import unittest +import unittest, sys try: - from unittest.mock import patch + from unittest.mock import patch, mock_open except ImportError: - from mock import patch + from mock import patch, mock_open import kernel_report @@ -18,10 +18,27 @@ class KernelReportGetFields(unittest.TestCase): """test operations involving getting existing values from a template machine""" + if sys.version_info.major == 3: + builtin_module_name = 'builtins' + else: + builtin_module_name = '__builtin__' + + @patch('{}.open'.format(builtin_module_name), new_callable=mock_open, read_data='1') + def test_safe_file_get_contents_fails_w_nonexist_file(self, mock_open): + mock_open.side_effect = IOError + rtn = kernel_report.safe_file_get_contents("junk") + self.assertIsNone(rtn) + @patch('kernel_report.safe_file_get_contents') def test_get_sysctl(self, mock): """do various tests of get_sysctl_fields""" - kernel_report.get_sysctl_fields() + my_value = kernel_report.get_sysctl_fields() + print(my_value) + + @patch('kernel_report.safe_file_get_contents') + @patch('re.findall') + def test_get_sysfs(self, mock_get_file, mock_re_search): + kernel_report.get_sysfs_fields() if __name__ == "__main__": diff --git a/tests/vars/tests_default.yml b/tests/vars/tests_default.yml index 2f7cc85f..768a4e67 100644 --- a/tests/vars/tests_default.yml +++ b/tests/vars/tests_default.yml @@ -13,4 +13,4 @@ __kernel_settings_test_sysctl: - name: vm.dirty_expire_centisecs value: '2000' - name: kernel.panic_on_oops - value: '1' \ No newline at end of file + value: '1' From 1e76323b44533ff7427190b78fef3c43af8cc389 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Wed, 28 Apr 2021 14:45:15 -0400 Subject: [PATCH 25/29] tweak the test for demo --- tests/tests_apply_kernel_report_settings.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index c0409104..c9006b43 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -31,10 +31,6 @@ index_var: my_idx run_once: true - - name: debug the host role - debug: - msg: "{{ host_role }}" - - name: Ensure pyudev package is installed on template machine package: name: "{{ __kernel_settings_test_pyudev_pkg }}" @@ -49,6 +45,7 @@ state: present reload: yes loop: "{{ __kernel_settings_test_sysctl }}" + when: host_role == "template" - name: get and store config values from template machine kernel_report: From 80a9cccb1926a3d3c45129cb74d9cf3a21577fdd Mon Sep 17 00:00:00 2001 From: mprovenc Date: Wed, 28 Apr 2021 14:45:35 -0400 Subject: [PATCH 26/29] add sysctl unit tests --- tests/unit/modules/test_kernel_report.py | 31 +++++++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/unit/modules/test_kernel_report.py b/tests/unit/modules/test_kernel_report.py index 67cb7b7c..1eec6863 100644 --- a/tests/unit/modules/test_kernel_report.py +++ b/tests/unit/modules/test_kernel_report.py @@ -15,6 +15,14 @@ import kernel_report +def dirty_sysctl_side_effect(*args, **kwargs): + if (args[0] == "/proc/sys/vm/dirty_background_bytes" or + args[0] == "/proc/sys/vm/dirty_background_ratio" or + args[0] == "/proc/sys/vm/dirty_bytes" or + args[0] == "/proc/sys/vm/dirty_ratio"): + return '0' + return None + class KernelReportGetFields(unittest.TestCase): """test operations involving getting existing values from a template machine""" @@ -24,16 +32,31 @@ class KernelReportGetFields(unittest.TestCase): builtin_module_name = '__builtin__' @patch('{}.open'.format(builtin_module_name), new_callable=mock_open, read_data='1') - def test_safe_file_get_contents_fails_w_nonexist_file(self, mock_open): + def test_safe_file_get_contents_fails_nonexist_file(self, mock_open): mock_open.side_effect = IOError rtn = kernel_report.safe_file_get_contents("junk") self.assertIsNone(rtn) @patch('kernel_report.safe_file_get_contents') - def test_get_sysctl(self, mock): + def test_get_sysctl_dirty_values_not_zero(self, mock_get_file): + """do various tests of get_sysctl_fields""" + mock_get_file.side_effect = dirty_sysctl_side_effect + sysctl = kernel_report.get_sysctl_fields() + self.assertEqual([], sysctl["kernel_settings_sysctl"]) + + @patch('kernel_report.safe_file_get_contents') + def test_get_sysctl_no_none_values(self, mock_get_file): """do various tests of get_sysctl_fields""" - my_value = kernel_report.get_sysctl_fields() - print(my_value) + mock_get_file.return_value = None + sysctl = kernel_report.get_sysctl_fields() + self.assertEqual([], sysctl["kernel_settings_sysctl"]) + + # test for kernel_settings_other to be empty when the files don't exist + + # test that regex is successful for transparent/defrag fields + + # test that when no cpus exist nothing is returned (tests for other modules as well) + @patch('kernel_report.safe_file_get_contents') @patch('re.findall') From b30a3376d07e6676d733b9c1a09a86335f51d8fb Mon Sep 17 00:00:00 2001 From: mprovenc Date: Wed, 28 Apr 2021 16:17:29 -0400 Subject: [PATCH 27/29] update documentation in kernel_report module --- library/kernel_report.py | 128 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/library/kernel_report.py b/library/kernel_report.py index 7712e0e8..cfb14551 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -6,6 +6,134 @@ # """ Generate kernel settings facts for a system """ +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +ANSIBLE_METADATA = { + "metadata_version": "1.1", + "status": ["preview"], + "supported_by": "community", +} + +DOCUMENTATION = """ +--- +module: kernel_report + +short_description: Report kernel settings from a template machine + +version_added: "2.9" + +description: + - | + Report kernel settings from a template machine and adds them to + ansible_facts. The current use case for this module is to be able to + apply kernel settings from a template machine to any number of targets; + particularly when admins may not be aware of which kernel settings they + have tweaked on the template machine and only know that they would like + to apply the kernel state to other machines. This module selectively + loops through sysctl and sysfs kernel settings to generate a "report" on + the kernel state of the template machine. The settings that it loops + through correspond to commonly changed fields in various tuned profiles. + Some of these commonly changed settings are device-specific, meaning that + they are tied to particular devices on the template machine, such as + cpus, usb interfaces, and graphics cards. These device-specific settings + include their corresponding device somewhere in the path name, thus + making it difficult and unreliable to apply directly to a target. To + address this issue, such settings are placed into a separate dictionary + called kernel_settings_device_specific. The dictionaries entitled + kernel_settings_sysfs and kernel_settings_sysctl contain settings which + can be applied directly their corresponding sysfs and sysctl tuned + plugins. The dictionary kernel_settings_other and its children contain + those kernel_settings which correspond to other tuned plugins such as + selinux, cpu, and vm. + - HORIZONTALLINE + - | + Note that if a particular kernel setting file does not exist on the template + system, that setting will not be included in the output of the report. + +author: + - Mary Provencher (@mprovenc) +""" + +EXAMPLES = """ +# register settings from a template machine +- name: get and store config values from template machine + kernel_report: + register: template_settings +""" + +RETURN = """ +# An example return. +ansible_facts: + description: Facts to add to ansible_facts. + returned: always + type: dict + contains: + kernel_settings_device_specific: + description: Device-specific kernel settings. + type: dict + returned: always + sample: + kernel_settings_cpu_governor: + - device: 'cpu0' + value: null + kernel_settings_disk_elevator: + - device: 'sr0' + value: 'bfq' + kernel_settings_disk_read_ahead_kb: + - device: 'sr0' + value: '128' + - device: 'vda' + value: '128' + kernel_settings_disk_scheduler_quantum: + - device: 'sr0' + value: null + kernel_settings_sampling_down_factor: + - device: 'cpu0' + value: null + kernel_settings_scsi_host_alpm: + - device: 'host0' + value: null + kernel_settings_other: + description: Kernel settings for various tuned plugins. + type: dict + returned: always + sample: + kernel_settings_cpu: + settings: + kernel_settings_net: + settings: + kernel_settings_selinux: + settings: + - name: 'avc_cache_threshold' + value:'512' + kernel_settings_vm: + settings: + - name: 'transparent_hugepage' + value: 'madvise' + - name: 'transparent_hugepage.defrag' + value: 'madvise' + kernel_settings_sysctl: + description: Sysctl kernel settings. + type: dict + returned: always + sample: + - name: 'fs.aio-max-nr' + value: '70000' + - name: 'fs.file-max' + value: '100000' + - name: 'vm.dirty_ratio' + value: '20' + kernel_settings_sysfs: + description: Sysfs settings that don't correspond to a particular tuned plugin + type: dict + returned: always + sample: + - name: '/sys/kernel/mm/ksm/run' + value: '0' +""" + import re import pyudev from ansible.module_utils.basic import AnsibleModule From a0d8822ac1fa8811d736db4500a3ba749a4b44f7 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Thu, 29 Apr 2021 10:58:57 -0400 Subject: [PATCH 28/29] update defaults documentation --- defaults/main.yml | 130 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 120 insertions(+), 10 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index c27164f1..4fd394e2 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -25,6 +25,126 @@ kernel_settings_sysctl: [] # value: 0 kernel_settings_sysfs: [] +# This is a `dict` of the settings that fall under tuned's cpu plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and`value`. For example: +# kernel_settings_cpu: +# devices: +# - cpu0 +# cpu1 +# settings: +# - name: governor +# value: powersave +# - name: min_perf_pct +# value: 20 +kernel_settings_cpu: null + +# This is a `dict` of the settings that fall under tuned's disk plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_disk: +# devices: +# settings: +# - name: elevator +# value: bfq +# - name: read_ahead_kb +# value: 256 +# - name: scheduler_quantum +# value: 64 +kernel_settings_disk: null + +# This is a `dict` of the settings that fall under tuned's net plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_net: +# devices: +# settings: +# - name: nf_conntrack_hashsize +# value: 1048576 +kernel_settings_net: null + +# This is a `dict` of the settings that fall under tuned's audio plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_audio: +# devices: +# settings: +# - name: timeout +# value: 10 +# - name: reset_controller +# value: 1 +kernel_settings_audio: null + +# This is a `dict` of the settings that fall under tuned's scsi_host plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_scsi_host: +# devices: +# settings: +# - name: alpm +# value: "min_power" +kernel_settings_scsi_host: null + +# This is a `dict` of the settings that fall under tuned's video plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_video: +# devices: +# settings: +# - name: radeon_powersave +# value: "auto" +kernel_settings_video: null + +# This is a `dict` of the settings that fall under tuned's usb plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_usb: +# devices: +# settings: +# - name: autosuspend +# value: 1 +kernel_settings_usb: null + +# This is a `dict` of the settings that fall under tuned's selinux plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_selinux: +# devices:s +# settings: +# - name: avc_cache_threshold +# value: 180 +kernel_settings_selinux: null + +# This is a `dict` of the settings that fall under tuned's vm plugin. The +# first item in the dict will be `devices`, which specifies the devices (if +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of +# `dict` items with allowed key names of `name` and `value`. For example: +# kernel_settings_vm: +# devices: +# settings: +# - name: tranmadvise +# - name: tsparent_hugepages +# value: ransparent_hugepages.defrag +# value: madvise +kernel_settings_vm: null + # A space delimited list of cpu numbers. # See systemd-system.conf man page - CPUAffinity kernel_settings_systemd_cpu_affinity: null @@ -38,16 +158,6 @@ kernel_settings_transparent_hugepages: null # value. The actual supported values may be different depending on your OS. kernel_settings_transparent_hugepages_defrag: null -kernel_settings_cpu: [] -kernel_settings_disk: [] -kernel_settings_net: [] -kernel_settings_audio: [] -kernel_settings_scsi_host: [] -kernel_settings_video: [] -kernel_settings_usb: [] -kernel_settings_selinux: [] -kernel_settings_vm: [] - # If purge is true, completely wipe out whatever the current settings # are and replace them with kernel_settings_parameters kernel_settings_purge: false From 8becea7e5f6b45f8845ce4e748fb7902d713ff16 Mon Sep 17 00:00:00 2001 From: mprovenc Date: Fri, 30 Apr 2021 15:51:27 -0400 Subject: [PATCH 29/29] reformat, finish unit tests --- defaults/main.yml | 58 +-- library/kernel_report.py | 445 ++++++++++++++----- tasks/main.yml | 67 +-- tests/tests_apply_kernel_report_settings.yml | 43 +- tests/unit/modules/test_kernel_report.py | 72 ++- 5 files changed, 472 insertions(+), 213 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 4fd394e2..a0020ae9 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -25,10 +25,10 @@ kernel_settings_sysctl: [] # value: 0 kernel_settings_sysfs: [] -# This is a `dict` of the settings that fall under tuned's cpu plugin. The +# This is a `dict` of the settings that fall under tuned's cpu plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and`value`. For example: # kernel_settings_cpu: # devices: @@ -41,10 +41,10 @@ kernel_settings_sysfs: [] # value: 20 kernel_settings_cpu: null -# This is a `dict` of the settings that fall under tuned's disk plugin. The +# This is a `dict` of the settings that fall under tuned's disk plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_disk: # devices: @@ -57,10 +57,10 @@ kernel_settings_cpu: null # value: 64 kernel_settings_disk: null -# This is a `dict` of the settings that fall under tuned's net plugin. The +# This is a `dict` of the settings that fall under tuned's net plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_net: # devices: @@ -69,10 +69,10 @@ kernel_settings_disk: null # value: 1048576 kernel_settings_net: null -# This is a `dict` of the settings that fall under tuned's audio plugin. The +# This is a `dict` of the settings that fall under tuned's audio plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_audio: # devices: @@ -83,10 +83,10 @@ kernel_settings_net: null # value: 1 kernel_settings_audio: null -# This is a `dict` of the settings that fall under tuned's scsi_host plugin. The -# first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# This is a `dict` of the settings that fall under tuned's scsi_host plugin. +# The first item in the dict will be `devices`, which specifies the devices +# (if any) to apply the settings to. The default is to apply to all devices. +# The second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_scsi_host: # devices: @@ -95,10 +95,10 @@ kernel_settings_audio: null # value: "min_power" kernel_settings_scsi_host: null -# This is a `dict` of the settings that fall under tuned's video plugin. The +# This is a `dict` of the settings that fall under tuned's video plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_video: # devices: @@ -107,10 +107,10 @@ kernel_settings_scsi_host: null # value: "auto" kernel_settings_video: null -# This is a `dict` of the settings that fall under tuned's usb plugin. The +# This is a `dict` of the settings that fall under tuned's usb plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_usb: # devices: @@ -119,22 +119,22 @@ kernel_settings_video: null # value: 1 kernel_settings_usb: null -# This is a `dict` of the settings that fall under tuned's selinux plugin. The +# This is a `dict` of the settings that fall under tuned's selinux plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_selinux: -# devices:s +# devices: # settings: # - name: avc_cache_threshold # value: 180 kernel_settings_selinux: null -# This is a `dict` of the settings that fall under tuned's vm plugin. The +# This is a `dict` of the settings that fall under tuned's vm plugin. The # first item in the dict will be `devices`, which specifies the devices (if -# any) to apply the settings to. The default is to apply to all devices. The -# second item in the dict will be `settings`, which consists of a list of +# any) to apply the settings to. The default is to apply to all devices. The +# second item in the dict will be `settings`, which consists of a list of # `dict` items with allowed key names of `name` and `value`. For example: # kernel_settings_vm: # devices: diff --git a/library/kernel_report.py b/library/kernel_report.py index cfb14551..026882d0 100644 --- a/library/kernel_report.py +++ b/library/kernel_report.py @@ -6,7 +6,7 @@ # """ Generate kernel settings facts for a system """ -from __future__ import (absolute_import, division, print_function) +from __future__ import absolute_import, division, print_function __metaclass__ = type @@ -27,30 +27,31 @@ description: - | Report kernel settings from a template machine and adds them to - ansible_facts. The current use case for this module is to be able to + ansible_facts. The current use case for this module is to be able to apply kernel settings from a template machine to any number of targets; particularly when admins may not be aware of which kernel settings they have tweaked on the template machine and only know that they would like to apply the kernel state to other machines. This module selectively loops through sysctl and sysfs kernel settings to generate a "report" on - the kernel state of the template machine. The settings that it loops + the kernel state of the template machine. The settings that it loops through correspond to commonly changed fields in various tuned profiles. Some of these commonly changed settings are device-specific, meaning that - they are tied to particular devices on the template machine, such as + they are tied to particular devices on the template machine, such as cpus, usb interfaces, and graphics cards. These device-specific settings - include their corresponding device somewhere in the path name, thus - making it difficult and unreliable to apply directly to a target. To - address this issue, such settings are placed into a separate dictionary - called kernel_settings_device_specific. The dictionaries entitled + include their corresponding device somewhere in the path name, thus + making it difficult and unreliable to apply directly to a target. To + address this issue, such settings are placed into a separate dictionary + called kernel_settings_device_specific. The dictionaries entitled kernel_settings_sysfs and kernel_settings_sysctl contain settings which - can be applied directly their corresponding sysfs and sysctl tuned + can be applied directly their corresponding sysfs and sysctl tuned plugins. The dictionary kernel_settings_other and its children contain those kernel_settings which correspond to other tuned plugins such as selinux, cpu, and vm. - HORIZONTALLINE - | - Note that if a particular kernel setting file does not exist on the template - system, that setting will not be included in the output of the report. + Note that if a particular kernel setting file does not exist on the + template system, that setting will not be included in the output of the + report. author: - Mary Provencher (@mprovenc) @@ -126,7 +127,7 @@ - name: 'vm.dirty_ratio' value: '20' kernel_settings_sysfs: - description: Sysfs settings that don't correspond to a particular tuned plugin + description: Sysfs settings (no particular tuned plugin) type: dict returned: always sample: @@ -138,53 +139,55 @@ import pyudev from ansible.module_utils.basic import AnsibleModule -SYSCTL_FIELDS = ['fs.aio-max-nr', - 'fs.file-max', - 'fs.inotify.max_user_watches', - 'kernel.hung_task_timeout_secs', - 'kernel.nmi_watchdog', - 'kernel.numa_balancing', - 'kernel.panic_on_oops', - 'kernel.pid_max', - 'kernel.printk', - 'kernel.sched_autogroup_enabled', - 'kernel.sched_latency_ns', - 'kernel.sched_migration_cost_ns', - 'kernel.sched_min_granularity_ns', - 'kernel.sched_rt_runtime_us', - 'kernel.sched_wakeup_granularity_ns', - 'kernel.sem', - 'kernel.shmall', - 'kernel.shmmax', - 'kernel.shmmni', - 'kernel.timer_migration', - 'net.core.busy_poll', - 'net.core.busy_read', - 'net.core.rmem_default', - 'net.core.rmem_max', - 'net.core.wmem_default', - 'net.core.wmem_max', - 'net.ipv4.ip_local_port_range', - 'net.ipv4.tcp_fastopen', - 'net.ipv4.tcp_rmem', - 'net.ipv4.tcp_timestamps', - 'net.ipv4.tcp_window_scaling', - 'net.ipv4.tcp_wmem', - 'net.ipv4.udp_mem', - 'net.netfilter.nf_conntrack_max', - 'vm.dirty_background_bytes', - 'vm.dirty_background_ratio', - 'vm.dirty_bytes', - 'vm.dirty_expire_centisecs', - 'vm.dirty_ratio', - 'vm.dirty_writeback_centisecs', - 'vm.hugepages_treat_as_movable', - 'vm.laptop_mode', - 'vm.max_map_count', - 'vm.min_free_kbytes', - 'vm.stat_interval', - 'vm.swappiness', - 'vm.zone_reclaim_mode'] +SYSCTL_FIELDS = [ + "fs.aio-max-nr", + "fs.file-max", + "fs.inotify.max_user_watches", + "kernel.hung_task_timeout_secs", + "kernel.nmi_watchdog", + "kernel.numa_balancing", + "kernel.panic_on_oops", + "kernel.pid_max", + "kernel.printk", + "kernel.sched_autogroup_enabled", + "kernel.sched_latency_ns", + "kernel.sched_migration_cost_ns", + "kernel.sched_min_granularity_ns", + "kernel.sched_rt_runtime_us", + "kernel.sched_wakeup_granularity_ns", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.timer_migration", + "net.core.busy_poll", + "net.core.busy_read", + "net.core.rmem_default", + "net.core.rmem_max", + "net.core.wmem_default", + "net.core.wmem_max", + "net.ipv4.ip_local_port_range", + "net.ipv4.tcp_fastopen", + "net.ipv4.tcp_rmem", + "net.ipv4.tcp_timestamps", + "net.ipv4.tcp_window_scaling", + "net.ipv4.tcp_wmem", + "net.ipv4.udp_mem", + "net.netfilter.nf_conntrack_max", + "vm.dirty_background_bytes", + "vm.dirty_background_ratio", + "vm.dirty_bytes", + "vm.dirty_expire_centisecs", + "vm.dirty_ratio", + "vm.dirty_writeback_centisecs", + "vm.hugepages_treat_as_movable", + "vm.laptop_mode", + "vm.max_map_count", + "vm.min_free_kbytes", + "vm.stat_interval", + "vm.swappiness", + "vm.zone_reclaim_mode", +] def safe_file_get_contents(filename): @@ -192,16 +195,27 @@ def safe_file_get_contents(filename): with open(filename) as f: return f.read().rstrip() except IOError as e: - print('safe_file_get_contents: suppressed exception FileNotFoundError with content \'%s\'' % e) + print( + "safe_file_get_contents: suppressed exception FileNotFoundError \ + with content '%s'" + % e + ) def get_sysctl_fields(): sysctl = [] for setting in SYSCTL_FIELDS: temp_dict = {"name": setting} - temp_dict["value"] = safe_file_get_contents("/proc/sys/%s" % setting.replace(".", "/")) + temp_dict["value"] = safe_file_get_contents( + "/proc/sys/%s" % setting.replace(".", "/") + ) if temp_dict["value"]: - if (setting == "vm.dirty_background_bytes" or setting == "vm.dirty_background_ratio" or setting == "vm.dirty_ratio" or setting == "vm.dirty_bytes") and temp_dict["value"] == "0": + if ( + setting == "vm.dirty_background_bytes" + or setting == "vm.dirty_background_ratio" + or setting == "vm.dirty_ratio" + or setting == "vm.dirty_bytes" + ) and temp_dict["value"] == "0": continue else: sysctl.append(temp_dict) @@ -223,24 +237,81 @@ def get_sysfs_fields(): kernel_settings_vm = {"settings": []} if safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct"): - kernel_settings_cpu["settings"].append({'name': 'min_perf_pct', 'value': safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/min_perf_pct")}) + kernel_settings_cpu["settings"].append( + { + "name": "min_perf_pct", + "value": safe_file_get_contents( + "/sys/devices/system/cpu/intel_pstate/min_perf_pct" + ), + } + ) if safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct"): - kernel_settings_cpu["settings"].append({'name': 'max_perf_pct', 'value': safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/max_perf_pct")}) + kernel_settings_cpu["settings"].append( + { + "name": "max_perf_pct", + "value": safe_file_get_contents( + "/sys/devices/system/cpu/intel_pstate/max_perf_pct" + ), + } + ) if safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo"): - kernel_settings_cpu["settings"].append({'name': 'no_turbo', 'value': safe_file_get_contents("/sys/devices/system/cpu/intel_pstate/no_turbo")}) + kernel_settings_cpu["settings"].append( + { + "name": "no_turbo", + "value": safe_file_get_contents( + "/sys/devices/system/cpu/intel_pstate/no_turbo" + ), + } + ) if safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize"): - kernel_settings_net["settings"].append({'name': 'nf_conntrack_hashsize', 'value': safe_file_get_contents("/sys/module/nf_conntrack/parameters/hashsize")}) + kernel_settings_net["settings"].append( + { + "name": "nf_conntrack_hashsize", + "value": safe_file_get_contents( + "/sys/module/nf_conntrack/parameters/hashsize" + ), + } + ) if safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold"): - kernel_settings_selinux["settings"].append({'name': 'avc_cache_threshold', 'value': safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold")}) + kernel_settings_selinux["settings"].append( + { + "name": "avc_cache_threshold", + "value": safe_file_get_contents("/sys/fs/selinux/avc/cache_threshold"), + } + ) if safe_file_get_contents("/sys/kernel/mm/transparent_hugepage/enabled"): - kernel_settings_vm["settings"].append({'name': 'transparent_hugepage', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/enabled'))[0]}) + kernel_settings_vm["settings"].append( + { + "name": "transparent_hugepage", + "value": re.findall( + r"\[(\w+)\]", + safe_file_get_contents( + "/sys/kernel/mm/transparent_hugepage/enabled" + ), + )[0], + } + ) if safe_file_get_contents("/sys/kernel/mm/transparent_hugepage/defrag"): - kernel_settings_vm["settings"].append({'name': 'transparent_hugepage.defrag', 'value': re.findall(r'\[(\w+)\]', safe_file_get_contents('/sys/kernel/mm/transparent_hugepage/defrag'))[0]}) - - kernel_settings_other["kernel_settings_cpu"] = kernel_settings_cpu - kernel_settings_other["kernel_settings_net"] = kernel_settings_net - kernel_settings_other["kernel_settings_selinux"] = kernel_settings_selinux - kernel_settings_other["kernel_settings_vm"] = kernel_settings_vm + kernel_settings_vm["settings"].append( + { + "name": "transparent_hugepage.defrag", + "value": re.findall( + r"\[(\w+)\]", + safe_file_get_contents( + "/sys/kernel/mm/transparent_hugepage/defrag" + ), + )[0], + } + ) + + if kernel_settings_cpu["settings"]: + kernel_settings_other["kernel_settings_cpu"] = kernel_settings_cpu + if kernel_settings_net["settings"]: + kernel_settings_other["kernel_settings_net"] = kernel_settings_net + if kernel_settings_selinux["settings"]: + kernel_settings_other["kernel_settings_selinux"] = kernel_settings_selinux + if kernel_settings_vm["settings"]: + kernel_settings_other["kernel_settings_vm"] = kernel_settings_vm result["kernel_settings_other"] = kernel_settings_other @@ -248,94 +319,236 @@ def get_sysfs_fields(): cpus = pyudev.Context().list_devices(subsystem="cpu") num_cpus = 0 - kernel_settings_device_specific["kernel_settings_cpu_governor"] = [] - kernel_settings_device_specific["kernel_settings_sampling_down_factor"] = [] + kernel_settings_cpu_governor = [] + kernel_settings_sampling_down_factor = [] for cpu in cpus: num_cpus += 1 - kernel_settings_device_specific["kernel_settings_cpu_governor"].append({'device': cpu.sys_name, 'value': safe_file_get_contents('%s/cpufreq/scaling_governor' % cpu.sys_path)}) - kernel_settings_device_specific["kernel_settings_sampling_down_factor"].append({'device': cpu.sys_name, 'value': safe_file_get_contents('/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor' % kernel_settings_device_specific['kernel_settings_cpu_governor'][-1])}) - - print("get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus) + kernel_settings_cpu_governor.append( + { + "device": cpu.sys_name, + "value": safe_file_get_contents( + "%s/cpufreq/scaling_governor" % cpu.sys_path + ), + } + ) + kernel_settings_sampling_down_factor.append( + { + "device": cpu.sys_name, + "value": safe_file_get_contents( + "/sys/devices/system/cpu/cpufreq/%s/sampling_down_factor" + % kernel_settings_cpu_governor[-1] + ), + } + ) + + if kernel_settings_cpu_governor: + kernel_settings_device_specific[ + "kernel_settings_cpu_governor" + ] = kernel_settings_cpu_governor + if kernel_settings_sampling_down_factor: + kernel_settings_device_specific[ + "kernel_settings_sampling_down_factor" + ] = kernel_settings_sampling_down_factor + + print( + "get_sysfs_fields: found %d cpus associated to the template machine" % num_cpus + ) # will collect a list of blocks associated to the template machine, but only use settings from the first one blocks = pyudev.Context().list_devices(subsystem="block") num_blocks = 0 - kernel_settings_device_specific["kernel_settings_disk_elevator"] = [] - kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"] = [] - kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"] = [] + kernel_settings_disk_elevator = [] + kernel_settings_disk_read_ahead_kb = [] + kernel_settings_disk_scheduler_quantum = [] for block in blocks: num_blocks += 1 schedulers = safe_file_get_contents("%s/queue/scheduler" % block.sys_path) if schedulers: - settings = re.findall(r'\[(\w+)\]', schedulers) + settings = re.findall(r"\[(\w+)\]", schedulers) if settings: - kernel_settings_device_specific["kernel_settings_disk_elevator"].append({'device': block.sys_name, 'value': settings[0]}) - kernel_settings_device_specific["kernel_settings_disk_read_ahead_kb"].append({'device': block.sys_name, 'value': safe_file_get_contents("%s/queue/read_ahead_kb" % block.sys_path)}) - kernel_settings_device_specific["kernel_settings_disk_scheduler_quantum"].append({'device': block.sys_name, 'value': safe_file_get_contents("%s/queue/iosched/quantum" % block.sys_path)}) - - print("get_sysfs_fields: found %d blocks associated to the template machine" % num_blocks) + kernel_settings_disk_elevator.append( + {"device": block.sys_name, "value": settings[0]} + ) + kernel_settings_disk_read_ahead_kb.append( + { + "device": block.sys_name, + "value": safe_file_get_contents( + "%s/queue/read_ahead_kb" % block.sys_path + ), + } + ) + kernel_settings_disk_scheduler_quantum.append( + { + "device": block.sys_name, + "value": safe_file_get_contents( + "%s/queue/iosched/quantum" % block.sys_path + ), + } + ) + + if kernel_settings_disk_elevator: + kernel_settings_device_specific[ + "kernel_settings_disk_elevator" + ] = kernel_settings_disk_elevator + if kernel_settings_disk_read_ahead_kb: + kernel_settings_device_specific[ + "kernel_settings_disk_read_ahead_kb" + ] = kernel_settings_disk_read_ahead_kb + if kernel_settings_disk_scheduler_quantum: + kernel_settings_device_specific[ + "kernel_settings_disk_scheduler_quantum" + ] = kernel_settings_disk_scheduler_quantum + + print( + "get_sysfs_fields: found %d blocks associated to the template machine" + % num_blocks + ) # will collect a list of sound cards associated to the template machine, but only use settings from the first one num_sound_cards = 0 - sound_cards = pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") + sound_cards = ( + pyudev.Context().list_devices(subsystem="sound").match_sys_name("card*") + ) - kernel_settings_device_specific["kernel_settings_audio_timeout"] = [] - kernel_settings_device_specific["kernel_settings_audio_reset_controller"] = [] + kernel_settings_audio_timeout = [] + kernel_settings_audio_reset_controller = [] for sound_card in sound_cards: module_name = sound_card.parent.driver if module_name in ["snd_hda_intel", "snd_ac97_codec"]: num_sound_cards += 1 - kernel_settings_device_specific["kernel_settings_audio_timeout"].append({'device': sound_card.sys_name, 'value': safe_file_get_contents("/sys/module/%s/parameters/power_save" % module_name)}) - kernel_settings_device_specific["kernel_settings_audio_reset_controller"].append({'device': sound_card.sys_name, 'value': safe_file_get_contents("/sys/module/%s/parameters/power_save_controller" % module_name)}) - - print("get_sysfs_fields: found %d sound modules associated to the template machine" % num_sound_cards) + kernel_settings_audio_timeout.append( + { + "device": sound_card.sys_name, + "value": safe_file_get_contents( + "/sys/module/%s/parameters/power_save" % module_name + ), + } + ) + kernel_settings_audio_reset_controller.append( + { + "device": sound_card.sys_name, + "value": safe_file_get_contents( + "/sys/module/%s/parameters/power_save_controller" % module_name + ), + } + ) + + if kernel_settings_audio_timeout: + kernel_settings_device_specific[ + "kernel_settings_audio_timeout" + ] = kernel_settings_audio_timeout + if kernel_settings_audio_reset_controller: + kernel_settings_device_specific[ + "kernel_settings_audio_reset_controller" + ] = kernel_settings_audio_reset_controller + + print( + "get_sysfs_fields: found %d sound modules associated to the template machine" + % num_sound_cards + ) # will collect a list of scsis associated to the template machine, but only use settings from the first one num_scsis = 0 scsis = pyudev.Context().list_devices(subsystem="scsi") - kernel_settings_device_specific["kernel_settings_scsi_host_alpm"] = [] + kernel_settings_scsi_host_alpm = [] for scsi in scsis: num_scsis += 1 - kernel_settings_device_specific["kernel_settings_scsi_host_alpm"].append({'device': scsi.sys_name, 'value': safe_file_get_contents("%s/link_power_management_policy" % scsi.sys_path)}) - - print("get_sysfs_fields: found %d scsis associated to the template machine" % num_scsis) + kernel_settings_scsi_host_alpm.append( + { + "device": scsi.sys_name, + "value": safe_file_get_contents( + "%s/link_power_management_policy" % scsi.sys_path + ), + } + ) + + if kernel_settings_scsi_host_alpm: + kernel_settings_device_specific[ + "kernel_settings_scsi_host_alpm" + ] = kernel_settings_scsi_host_alpm + print( + "get_sysfs_fields: found %d scsis associated to the template machine" + % num_scsis + ) # will collect a list of graphics cards associated to the template machine, but only use settings from the first one num_gcards = 0 - gcards = pyudev.Context().list_devices(subsystem="drm").match_sys_name("card*").match_property("DEVTYPE", "drm_minor") + gcards = ( + pyudev.Context() + .list_devices(subsystem="drm") + .match_sys_name("card*") + .match_property("DEVTYPE", "drm_minor") + ) - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"] = [] + kernel_settings_video_radeon_powersave = [] for gcard in gcards: num_gcards += 1 method = safe_file_get_contents("%s/device/power_method" % gcard.sys_path) if method == "profile": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device': gcard.sys_name, 'value': safe_file_get_contents("%s/device/power_profile" % gcard.sys_path)}) + kernel_settings_video_radeon_powersave.append( + { + "device": gcard.sys_name, + "value": safe_file_get_contents( + "%s/device/power_profile" % gcard.sys_path + ), + } + ) elif method == "dynpm": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device': gcard.sys_name, 'value': 'dynpm'}) + kernel_settings_video_radeon_powersave.append( + {"device": gcard.sys_name, "value": "dynpm"} + ) elif method == "dpm": - kernel_settings_device_specific["kernel_settings_video_radeon_powersave"].append({'device': gcard.sys_name, 'value': "dpm-%s" % safe_file_get_contents("%s/device/power_dpm_state" % gcard.sys_path)}) - - print("get_sysfs_fields: found %d gcards associated to the template machine" % num_gcards) + kernel_settings_video_radeon_powersave.append( + { + "device": gcard.sys_name, + "value": "dpm-%s" + % safe_file_get_contents( + "%s/device/power_dpm_state" % gcard.sys_path + ), + } + ) + + if kernel_settings_video_radeon_powersave: + kernel_settings_device_specific[ + "kernel_settings_video_radeon_powersave" + ] = kernel_settings_video_radeon_powersave + + print( + "get_sysfs_fields: found %d gcards associated to the template machine" + % num_gcards + ) # will collect a list of usb interfaces associated to the template machine, but only use settings from the first one num_usbs = 0 usbs = pyudev.Context().list_devices(subsystem="usb") - kernel_settings_device_specific["kernel_settings_usb_autosuspend"] = [] + kernel_settings_usb_autosuspend = [] for usb in usbs: num_usbs += 1 - kernel_settings_device_specific["kernel_settings_usb_autosuspend"].append({'device': usb.sys_name, 'value': safe_file_get_contents("%s/power/autosuspend" % usb.sys_path)}) + kernel_settings_usb_autosuspend.append( + { + "device": usb.sys_name, + "value": safe_file_get_contents("%s/power/autosuspend" % usb.sys_path), + } + ) + + if kernel_settings_usb_autosuspend: + kernel_settings_device_specific[ + "kernel_settings_usb_autosuspend" + ] = kernel_settings_usb_autosuspend result["kernel_settings_device_specific"] = kernel_settings_device_specific - print("get_sysfs_fields: found %d usbs associated to the template machine" % num_usbs) + print( + "get_sysfs_fields: found %d usbs associated to the template machine" % num_usbs + ) other_sysfs = [] ksm_dict = {"name": "/sys/kernel/mm/ksm/run"} @@ -361,17 +574,13 @@ def run_module(): ansible_facts=dict(), ) - module = AnsibleModule( - argument_spec=module_args, - supports_check_mode=True - ) + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) if module.check_mode: module.exit_json(**result) - # result['ansible_facts'] = {**get_sysfs_fields(), **get_sysctl_fields()} - result['ansible_facts'] = get_sysfs_fields() - result['ansible_facts'].update(get_sysctl_fields()) + result["ansible_facts"] = get_sysfs_fields() + result["ansible_facts"].update(get_sysctl_fields()) module.exit_json(**result) @@ -380,5 +589,5 @@ def main(): run_module() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tasks/main.yml b/tasks/main.yml index 03adcdb7..dea1e084 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -41,42 +41,47 @@ - name: Apply kernel settings kernel_settings: - cpu: "{{ kernel_settings_cpu.settings | - union([{'name': 'devices',\ - 'value': kernel_settings_cpu.devices | - d('*', true)}]) if kernel_settings_cpu else omit }}" - disk: "{{ kernel_settings_disk.settings | - union([{'name': 'devices',\ + cpu: "{{ kernel_settings_cpu is defined | + ternary(kernel_settings_cpu.settings | d([]) | + union([{'name': 'devices', 'value': kernel_settings_cpu.devices | + d('*', true)}]) | d({}), omit) }}" + disk: "{{ kernel_settings_disk is defined | + ternary(kernel_settings_disk.settings | d([]) | + union([{'name': 'devices', \ 'value': kernel_settings_disk.devices | - d('*', true)}]) if kernel_settings_disk else omit }}" - net: "{{ kernel_settings_net.settings | - union([{'name': 'devices',\ - 'value': kernel_settings_net.devices | - d('*', true)}]) if kernel_settings_net else omit }}" - audio: "{{ kernel_settings_audio.settings | - union([{'name': 'devices',\ + d('*', true)}]) | d({}), omit) }}" + net: "{{ kernel_settings_net is defined | + ternary(kernel_settings_net.settings | d([]) | + union([{'name': 'devices', 'value': kernel_settings_net.devices | + d('*', true)}]) | d({}), omit) }}" + audio: "{{ kernel_settings_audio is defined | + ternary(kernel_settings_audio.settings | d([]) | + union([{'name': 'devices', \ 'value': kernel_settings_audio.devices | - d('*', true)}]) if kernel_settings_audio else omit }}" - scsi_host: "{{ kernel_settings_scsi_host.settings | - union([{'name': 'devices',\ + d('*', true)}]) | d({}), omit) }}" + scsi_host: "{{ kernel_settings_scsi_host is defined | + ternary(kernel_settings_scsi_host.settings | d([]) | + union([{'name': 'devices', \ 'value': kernel_settings_scsi_host.devices | - d('*', true)}]) if kernel_settings_scsi_host else omit }}" - video: "{{ kernel_settings_video.settings | - union([{'name': 'devices',\ + d('*', true)}]) | d({}), omit) }}" + video: "{{ kernel_settings_video is defined | + ternary(kernel_settings_video.settings | d([]) | + union([{'name': 'devices', \ 'value': kernel_settings_video.devices | - d('*', true)}]) if kernel_settings_video else omit }}" - usb: "{{ kernel_settings_usb.settings | - union([{'name': 'devices',\ - 'value': kernel_settings_usb.devices | - d('*', true)}]) if kernel_settings_usb else omit }}" - selinux: "{{ kernel_settings_selinux.settings | - union([{'name': 'devices',\ + d('*', true)}]) | d({}), omit) }}" + usb: "{{ kernel_settings_usb is defined | + ternary(kernel_settings_usb.settings | d([]) | + union([{'name': 'devices', 'value': kernel_settings_usb.devices | + d('*', true)}]) | d({}), omit) }}" + selinux: "{{ kernel_settings_selinux is defined | + ternary(kernel_settings_selinux.settings | d([]) | + union([{'name': 'devices', \ 'value': kernel_settings_selinux.devices | - d('*', true)}]) if kernel_settings_selinux else omit }}" - vm: "{{ kernel_settings_vm.settings | - union([{'name': 'devices',\ - 'value': kernel_settings_vm.devices | - d('*', true)}]) if kernel_settings_vm else omit }}" + d('*', true)}]) | d({}), omit) }}" + vm: "{{ kernel_settings_vm is defined | + ternary(kernel_settings_vm.settings | d([]) | + union([{'name': 'devices', 'value': kernel_settings_vm.devices | + d('*', true)}]) | d({}), omit) }}" sysctl: "{{ kernel_settings_sysctl if kernel_settings_sysctl else omit }}" sysfs: "{{ kernel_settings_sysfs if kernel_settings_sysfs else omit }}" systemd: diff --git a/tests/tests_apply_kernel_report_settings.yml b/tests/tests_apply_kernel_report_settings.yml index c9006b43..6bcb753b 100644 --- a/tests/tests_apply_kernel_report_settings.yml +++ b/tests/tests_apply_kernel_report_settings.yml @@ -74,22 +74,39 @@ include_role: name: linux-system-roles.kernel_settings vars: - kernel_settings_cpu: "{{ hostvars[template_hostname].\ + kernel_settings_cpu: "{{ (hostvars[template_hostname].\ template_settings.ansible_facts.\ - kernel_settings_other.\ - kernel_settings_cpu }}" - kernel_settings_net: "{{ hostvars[template_hostname].\ + kernel_settings_other.kernel_settings_cpu) + if 'kernel_settings_cpu' in + hostvars[template_hostname].\ + template_settings.\ + ansible_facts.kernel_settings_other + else omit }}" + kernel_settings_net: "{{ (hostvars[template_hostname].\ template_settings.ansible_facts.\ - kernel_settings_other.\ - kernel_settings_net }}" - kernel_settings_selinux: "{{ hostvars[template_hostname].\ - template_settings.ansible_facts.\ - kernel_settings_other.\ - kernel_settings_selinux }}" - kernel_settings_vm: "{{ hostvars[template_hostname].\ + kernel_settings_other.kernel_settings_net) + if 'kernel_settings_net' in + hostvars[template_hostname].\ + template_settings.\ + ansible_facts.kernel_settings_other + else omit }}" + kernel_settings_selinux: "{{ (hostvars[template_hostname].\ + template_settings.ansible_facts.\ + kernel_settings_other.\ + kernel_settings_selinux) + if 'kernel_settings_selinux' in + hostvars[template_hostname].\ + template_settings.\ + ansible_facts.kernel_settings_other + else omit }}" + kernel_settings_vm: "{{ (hostvars[template_hostname].\ template_settings.ansible_facts.\ - kernel_settings_other.\ - kernel_settings_vm }}" + kernel_settings_other.kernel_settings_vm) + if 'kernel_settings_vm' in + hostvars[template_hostname].\ + template_settings.\ + ansible_facts.kernel_settings_other + else omit }}" kernel_settings_sysfs: "{{ hostvars[template_hostname].\ template_settings.ansible_facts.\ kernel_settings_sysfs }}" diff --git a/tests/unit/modules/test_kernel_report.py b/tests/unit/modules/test_kernel_report.py index 1eec6863..e6e98223 100644 --- a/tests/unit/modules/test_kernel_report.py +++ b/tests/unit/modules/test_kernel_report.py @@ -5,7 +5,8 @@ # """ Unit tests for kernel_report module """ -import unittest, sys +import unittest +import sys try: from unittest.mock import patch, mock_open @@ -13,55 +14,82 @@ from mock import patch, mock_open import kernel_report +import pyudev def dirty_sysctl_side_effect(*args, **kwargs): - if (args[0] == "/proc/sys/vm/dirty_background_bytes" or - args[0] == "/proc/sys/vm/dirty_background_ratio" or - args[0] == "/proc/sys/vm/dirty_bytes" or - args[0] == "/proc/sys/vm/dirty_ratio"): - return '0' + if ( + args[0] == "/proc/sys/vm/dirty_background_bytes" + or args[0] == "/proc/sys/vm/dirty_background_ratio" + or args[0] == "/proc/sys/vm/dirty_bytes" + or args[0] == "/proc/sys/vm/dirty_ratio" + ): + return "0" return None + +def regex_side_effect(*args, **kwargs): + if ( + args[0] == "/sys/kernel/mm/transparent_hugepage/enabled" + or args[0] == "/sys/kernel/mm/transparent_hugepage/defrag" + ): + return "always defer defer+madvise [madvise] never" + return None + + class KernelReportGetFields(unittest.TestCase): """test operations involving getting existing values from a template machine""" if sys.version_info.major == 3: - builtin_module_name = 'builtins' + builtin_module_name = "builtins" else: - builtin_module_name = '__builtin__' + builtin_module_name = "__builtin__" - @patch('{}.open'.format(builtin_module_name), new_callable=mock_open, read_data='1') + @patch("{}.open".format(builtin_module_name), new_callable=mock_open, read_data="1") def test_safe_file_get_contents_fails_nonexist_file(self, mock_open): mock_open.side_effect = IOError rtn = kernel_report.safe_file_get_contents("junk") self.assertIsNone(rtn) - @patch('kernel_report.safe_file_get_contents') + @patch("kernel_report.safe_file_get_contents") def test_get_sysctl_dirty_values_not_zero(self, mock_get_file): """do various tests of get_sysctl_fields""" mock_get_file.side_effect = dirty_sysctl_side_effect sysctl = kernel_report.get_sysctl_fields() self.assertEqual([], sysctl["kernel_settings_sysctl"]) - @patch('kernel_report.safe_file_get_contents') + @patch("kernel_report.safe_file_get_contents") def test_get_sysctl_no_none_values(self, mock_get_file): """do various tests of get_sysctl_fields""" mock_get_file.return_value = None sysctl = kernel_report.get_sysctl_fields() self.assertEqual([], sysctl["kernel_settings_sysctl"]) - # test for kernel_settings_other to be empty when the files don't exist - - # test that regex is successful for transparent/defrag fields - - # test that when no cpus exist nothing is returned (tests for other modules as well) - - - @patch('kernel_report.safe_file_get_contents') - @patch('re.findall') - def test_get_sysfs(self, mock_get_file, mock_re_search): - kernel_report.get_sysfs_fields() + @patch("kernel_report.safe_file_get_contents") + def test_get_sysfs_other_settings_empty(self, mock_get_file): + """test for kernel_settings_other to be empty when the files don't exist""" + mock_get_file.return_value = None + sysfs = kernel_report.get_sysfs_fields() + self.assertEqual({}, sysfs["kernel_settings_other"]) + + @patch("kernel_report.safe_file_get_contents") + def test_get_sysfs_regex_fields(self, mock_get_file): + """test that regex is successful for transparent/defrag fields""" + mock_get_file.side_effect = regex_side_effect + sysfs = kernel_report.get_sysfs_fields() + for setting in sysfs["kernel_settings_other"]["kernel_settings_vm"]["settings"]: + if ( + setting["name"] == "transparent_hugepage" + or setting["name"] == "transparent_hugepage.defrag" + ): + self.assertEqual("madvise", setting["value"]) + + @patch("kernel_report.pyudev.Context.list_devices") + def test_get_sysfs_no_devices(self, mock_list_devices): + """ensure that when no devices exist kernel_settings_device_specific is empty""" + mock_list_devices.return_value = pyudev.Context().list_devices(subsystem="") + sysfs = kernel_report.get_sysfs_fields() + self.assertEqual({}, sysfs["kernel_settings_device_specific"]) if __name__ == "__main__":