diff options
Diffstat (limited to 'roles')
3 files changed, 534 insertions, 0 deletions
diff --git a/roles/openshift_health_checker/library/etcdkeysize.py b/roles/openshift_health_checker/library/etcdkeysize.py new file mode 100644 index 000000000..620e82d87 --- /dev/null +++ b/roles/openshift_health_checker/library/etcdkeysize.py @@ -0,0 +1,122 @@ +#!/usr/bin/python +"""Ansible module that recursively determines if the size of a key in an etcd cluster exceeds a given limit.""" + +from ansible.module_utils.basic import AnsibleModule + + +try: + import etcd + + IMPORT_EXCEPTION_MSG = None +except ImportError as err: + IMPORT_EXCEPTION_MSG = str(err) + + from collections import namedtuple + EtcdMock = namedtuple("etcd", ["EtcdKeyNotFound"]) + etcd = EtcdMock(KeyError) + + +# pylint: disable=too-many-arguments +def check_etcd_key_size(client, key, size_limit, total_size=0, depth=0, depth_limit=1000, visited=None): + """Check size of an etcd path starting at given key. Returns tuple (string, bool)""" + if visited is None: + visited = set() + + if key in visited: + return 0, False + + visited.add(key) + + try: + result = client.read(key, recursive=False) + except etcd.EtcdKeyNotFound: + return 0, False + + size = 0 + limit_exceeded = False + + for node in result.leaves: + if depth >= depth_limit: + raise Exception("Maximum recursive stack depth ({}) exceeded.".format(depth_limit)) + + if size_limit and total_size + size > size_limit: + return size, True + + if not node.dir: + size += len(node.value) + continue + + key_size, limit_exceeded = check_etcd_key_size(client, node.key, + size_limit, + total_size + size, + depth + 1, + depth_limit, visited) + size += key_size + + max_limit_exceeded = limit_exceeded or (total_size + size > size_limit) + return size, max_limit_exceeded + + +def main(): # pylint: disable=missing-docstring,too-many-branches + module = AnsibleModule( + argument_spec=dict( + size_limit_bytes=dict(type="int", default=0), + paths=dict(type="list", default=["/openshift.io/images"]), + host=dict(type="str", default="127.0.0.1"), + port=dict(type="int", default=4001), + protocol=dict(type="str", default="http"), + version_prefix=dict(type="str", default=""), + allow_redirect=dict(type="bool", default=False), + cert=dict(type="dict", default=""), + ca_cert=dict(type="str", default=None), + ), + supports_check_mode=True + ) + + module.params["cert"] = ( + module.params["cert"]["cert"], + module.params["cert"]["key"], + ) + + size_limit = module.params.pop("size_limit_bytes") + paths = module.params.pop("paths") + + limit_exceeded = False + + try: + # pylint: disable=no-member + client = etcd.Client(**module.params) + except AttributeError as attrerr: + msg = str(attrerr) + if IMPORT_EXCEPTION_MSG: + msg = IMPORT_EXCEPTION_MSG + if "No module named etcd" in IMPORT_EXCEPTION_MSG: + # pylint: disable=redefined-variable-type + msg = ('Unable to import the python "etcd" dependency. ' + 'Make sure python-etcd is installed on the host.') + + module.exit_json( + failed=True, + changed=False, + size_limit_exceeded=limit_exceeded, + msg=msg, + ) + + return + + size = 0 + for path in paths: + path_size, limit_exceeded = check_etcd_key_size(client, path, size_limit - size) + size += path_size + + if limit_exceeded: + break + + module.exit_json( + changed=False, + size_limit_exceeded=limit_exceeded, + ) + + +if __name__ == '__main__': + main() diff --git a/roles/openshift_health_checker/openshift_checks/etcd_imagedata_size.py b/roles/openshift_health_checker/openshift_checks/etcd_imagedata_size.py new file mode 100644 index 000000000..c04a69765 --- /dev/null +++ b/roles/openshift_health_checker/openshift_checks/etcd_imagedata_size.py @@ -0,0 +1,84 @@ +""" +Ansible module for determining if the size of OpenShift image data exceeds a specified limit in an etcd cluster. +""" + +from openshift_checks import OpenShiftCheck, OpenShiftCheckException, get_var + + +class EtcdImageDataSize(OpenShiftCheck): + """Check that total size of OpenShift image data does not exceed the recommended limit in an etcd cluster""" + + name = "etcd_imagedata_size" + tags = ["etcd"] + + def run(self, tmp, task_vars): + etcd_mountpath = self._get_etcd_mountpath(get_var(task_vars, "ansible_mounts")) + etcd_avail_diskspace = etcd_mountpath["size_available"] + etcd_total_diskspace = etcd_mountpath["size_total"] + + etcd_imagedata_size_limit = get_var(task_vars, + "etcd_max_image_data_size_bytes", + default=int(0.5 * float(etcd_total_diskspace - etcd_avail_diskspace))) + + etcd_is_ssl = get_var(task_vars, "openshift", "master", "etcd_use_ssl", default=False) + etcd_port = get_var(task_vars, "openshift", "master", "etcd_port", default=2379) + etcd_hosts = get_var(task_vars, "openshift", "master", "etcd_hosts") + + config_base = get_var(task_vars, "openshift", "common", "config_base") + + cert = task_vars.get("etcd_client_cert", config_base + "/master/master.etcd-client.crt") + key = task_vars.get("etcd_client_key", config_base + "/master/master.etcd-client.key") + ca_cert = task_vars.get("etcd_client_ca_cert", config_base + "/master/master.etcd-ca.crt") + + for etcd_host in list(etcd_hosts): + args = { + "size_limit_bytes": etcd_imagedata_size_limit, + "paths": ["/openshift.io/images", "/openshift.io/imagestreams"], + "host": etcd_host, + "port": etcd_port, + "protocol": "https" if etcd_is_ssl else "http", + "version_prefix": "/v2", + "allow_redirect": True, + "ca_cert": ca_cert, + "cert": { + "cert": cert, + "key": key, + }, + } + + etcdkeysize = self.module_executor("etcdkeysize", args, task_vars) + + if etcdkeysize.get("rc", 0) != 0 or etcdkeysize.get("failed"): + msg = 'Failed to retrieve stats for etcd host "{host}": {reason}' + reason = etcdkeysize.get("msg") + if etcdkeysize.get("module_stderr"): + reason = etcdkeysize["module_stderr"] + + msg = msg.format(host=etcd_host, reason=reason) + return {"failed": True, "changed": False, "msg": msg} + + if etcdkeysize["size_limit_exceeded"]: + limit = self._to_gigabytes(etcd_imagedata_size_limit) + msg = ("The size of OpenShift image data stored in etcd host " + "\"{host}\" exceeds the maximum recommended limit of {limit:.2f} GB. " + "Use the `oadm prune images` command to cleanup unused Docker images.") + return {"failed": True, "msg": msg.format(host=etcd_host, limit=limit)} + + return {"changed": False} + + @staticmethod + def _get_etcd_mountpath(ansible_mounts): + valid_etcd_mount_paths = ["/var/lib/etcd", "/var/lib", "/var", "/"] + + mount_for_path = {mnt.get("mount"): mnt for mnt in ansible_mounts} + for path in valid_etcd_mount_paths: + if path in mount_for_path: + return mount_for_path[path] + + paths = ', '.join(sorted(mount_for_path)) or 'none' + msg = "Unable to determine a valid etcd mountpath. Paths mounted: {}.".format(paths) + raise OpenShiftCheckException(msg) + + @staticmethod + def _to_gigabytes(byte_size): + return float(byte_size) / 10.0**9 diff --git a/roles/openshift_health_checker/test/etcd_imagedata_size_test.py b/roles/openshift_health_checker/test/etcd_imagedata_size_test.py new file mode 100644 index 000000000..df9d52d41 --- /dev/null +++ b/roles/openshift_health_checker/test/etcd_imagedata_size_test.py @@ -0,0 +1,328 @@ +import pytest + +from collections import namedtuple +from openshift_checks.etcd_imagedata_size import EtcdImageDataSize, OpenShiftCheckException +from etcdkeysize import check_etcd_key_size + + +def fake_etcd_client(root): + fake_nodes = dict() + fake_etcd_node(root, fake_nodes) + + clientclass = namedtuple("client", ["read"]) + return clientclass(lambda key, recursive: fake_etcd_result(fake_nodes[key])) + + +def fake_etcd_result(fake_node): + resultclass = namedtuple("result", ["leaves"]) + if not fake_node.dir: + return resultclass([fake_node]) + + return resultclass(fake_node.leaves) + + +def fake_etcd_node(node, visited): + min_req_fields = ["dir", "key"] + fields = list(node) + leaves = [] + + if node["dir"] and node.get("leaves"): + for leaf in node["leaves"]: + leaves.append(fake_etcd_node(leaf, visited)) + + if len(set(min_req_fields) - set(fields)) > 0: + raise ValueError("fake etcd nodes require at least {} fields.".format(min_req_fields)) + + if node.get("leaves"): + node["leaves"] = leaves + + nodeclass = namedtuple("node", fields) + nodeinst = nodeclass(**node) + visited[nodeinst.key] = nodeinst + + return nodeinst + + +@pytest.mark.parametrize('ansible_mounts,extra_words', [ + ([], ['none']), # empty ansible_mounts + ([{'mount': '/mnt'}], ['/mnt']), # missing relevant mount paths +]) +def test_cannot_determine_available_mountpath(ansible_mounts, extra_words): + task_vars = dict( + ansible_mounts=ansible_mounts, + ) + check = EtcdImageDataSize(execute_module=fake_execute_module) + + with pytest.raises(OpenShiftCheckException) as excinfo: + check.run(tmp=None, task_vars=task_vars) + + for word in 'determine valid etcd mountpath'.split() + extra_words: + assert word in str(excinfo.value) + + +@pytest.mark.parametrize('ansible_mounts,tree,size_limit,should_fail,extra_words', [ + ( + # test that default image size limit evals to 1/2 * (total size in use) + [{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 80 * 10**9, + }], + {"dir": False, "key": "/", "value": "1234"}, + None, + False, + [], + ), + ( + [{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 48 * 10**9, + }], + {"dir": False, "key": "/", "value": "1234"}, + None, + False, + [], + ), + ( + # set max size limit for image data to be below total node value + # total node value is defined as the sum of the value field + # from every node + [{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 48 * 10**9, + }], + {"dir": False, "key": "/", "value": "12345678"}, + 7, + True, + ["exceeds the maximum recommended limit", "0.00 GB"], + ), + ( + [{ + 'mount': '/', + 'size_available': 48 * 10**9 - 1, + 'size_total': 48 * 10**9, + }], + {"dir": False, "key": "/", "value": "1234"}, + None, + True, + ["exceeds the maximum recommended limit", "0.00 GB"], + ) +]) +def test_check_etcd_key_size_calculates_correct_limit(ansible_mounts, tree, size_limit, should_fail, extra_words): + def execute_module(module_name, args, tmp=None, task_vars=None): + if module_name != "etcdkeysize": + return { + "changed": False, + } + + client = fake_etcd_client(tree) + s, limit_exceeded = check_etcd_key_size(client, tree["key"], args["size_limit_bytes"]) + + return {"size_limit_exceeded": limit_exceeded} + + task_vars = dict( + etcd_max_image_data_size_bytes=size_limit, + ansible_mounts=ansible_mounts, + openshift=dict( + master=dict(etcd_hosts=["localhost"]), + common=dict(config_base="/var/lib/origin") + ) + ) + if size_limit is None: + task_vars.pop("etcd_max_image_data_size_bytes") + + check = EtcdImageDataSize(execute_module=execute_module).run(tmp=None, task_vars=task_vars) + + if should_fail: + assert check["failed"] + + for word in extra_words: + assert word in check["msg"] + else: + assert not check.get("failed", False) + + +@pytest.mark.parametrize('ansible_mounts,tree,root_path,expected_size,extra_words', [ + ( + [{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 80 * 10**9, + }], + # test recursive size check on tree with height > 1 + { + "dir": True, + "key": "/", + "leaves": [ + {"dir": False, "key": "/foo1", "value": "1234"}, + {"dir": False, "key": "/foo2", "value": "1234"}, + {"dir": False, "key": "/foo3", "value": "1234"}, + {"dir": False, "key": "/foo4", "value": "1234"}, + { + "dir": True, + "key": "/foo5", + "leaves": [ + {"dir": False, "key": "/foo/bar1", "value": "56789"}, + {"dir": False, "key": "/foo/bar2", "value": "56789"}, + {"dir": False, "key": "/foo/bar3", "value": "56789"}, + { + "dir": True, + "key": "/foo/bar4", + "leaves": [ + {"dir": False, "key": "/foo/bar/baz1", "value": "123"}, + {"dir": False, "key": "/foo/bar/baz2", "value": "123"}, + ] + }, + ] + }, + ] + }, + "/", + 37, + [], + ), + ( + [{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 80 * 10**9, + }], + # test correct sub-tree size calculation + { + "dir": True, + "key": "/", + "leaves": [ + {"dir": False, "key": "/foo1", "value": "1234"}, + {"dir": False, "key": "/foo2", "value": "1234"}, + {"dir": False, "key": "/foo3", "value": "1234"}, + {"dir": False, "key": "/foo4", "value": "1234"}, + { + "dir": True, + "key": "/foo5", + "leaves": [ + {"dir": False, "key": "/foo/bar1", "value": "56789"}, + {"dir": False, "key": "/foo/bar2", "value": "56789"}, + {"dir": False, "key": "/foo/bar3", "value": "56789"}, + { + "dir": True, + "key": "/foo/bar4", + "leaves": [ + {"dir": False, "key": "/foo/bar/baz1", "value": "123"}, + {"dir": False, "key": "/foo/bar/baz2", "value": "123"}, + ] + }, + ] + }, + ] + }, + "/foo5", + 21, + [], + ), + ( + [{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 80 * 10**9, + }], + # test that a non-existing key is handled correctly + { + "dir": False, + "key": "/", + "value": "1234", + }, + "/missing", + 0, + [], + ), + ( + [{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 80 * 10**9, + }], + # test etcd cycle handling + { + "dir": True, + "key": "/", + "leaves": [ + {"dir": False, "key": "/foo1", "value": "1234"}, + {"dir": False, "key": "/foo2", "value": "1234"}, + {"dir": False, "key": "/foo3", "value": "1234"}, + {"dir": False, "key": "/foo4", "value": "1234"}, + { + "dir": True, + "key": "/", + "leaves": [ + {"dir": False, "key": "/foo1", "value": "1"}, + ], + }, + ] + }, + "/", + 16, + [], + ), +]) +def test_etcd_key_size_check_calculates_correct_size(ansible_mounts, tree, root_path, expected_size, extra_words): + def execute_module(module_name, args, tmp=None, task_vars=None): + if module_name != "etcdkeysize": + return { + "changed": False, + } + + client = fake_etcd_client(tree) + size, limit_exceeded = check_etcd_key_size(client, root_path, args["size_limit_bytes"]) + + assert size == expected_size + return { + "size_limit_exceeded": limit_exceeded, + } + + task_vars = dict( + ansible_mounts=ansible_mounts, + openshift=dict( + master=dict(etcd_hosts=["localhost"]), + common=dict(config_base="/var/lib/origin") + ) + ) + + check = EtcdImageDataSize(execute_module=execute_module).run(tmp=None, task_vars=task_vars) + assert not check.get("failed", False) + + +def test_etcdkeysize_module_failure(): + def execute_module(module_name, tmp=None, task_vars=None): + if module_name != "etcdkeysize": + return { + "changed": False, + } + + return { + "rc": 1, + "module_stderr": "failure", + } + + task_vars = dict( + ansible_mounts=[{ + 'mount': '/', + 'size_available': 40 * 10**9, + 'size_total': 80 * 10**9, + }], + openshift=dict( + master=dict(etcd_hosts=["localhost"]), + common=dict(config_base="/var/lib/origin") + ) + ) + + check = EtcdImageDataSize(execute_module=execute_module).run(tmp=None, task_vars=task_vars) + + assert check["failed"] + for word in "Failed to retrieve stats": + assert word in check["msg"] + + +def fake_execute_module(*args): + raise AssertionError('this function should not be called') |