diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/src/ooinstall/cli_installer.py | 79 | ||||
| -rw-r--r-- | utils/src/ooinstall/oo_config.py | 2 | ||||
| -rw-r--r-- | utils/src/ooinstall/openshift_ansible.py | 19 | ||||
| -rw-r--r-- | utils/src/ooinstall/variants.py | 3 | ||||
| -rw-r--r-- | utils/test/cli_installer_tests.py | 153 | ||||
| -rw-r--r-- | utils/test/fixture.py | 10 | 
6 files changed, 185 insertions, 81 deletions
| diff --git a/utils/src/ooinstall/cli_installer.py b/utils/src/ooinstall/cli_installer.py index c53ca7b18..dd9d517f1 100644 --- a/utils/src/ooinstall/cli_installer.py +++ b/utils/src/ooinstall/cli_installer.py @@ -666,7 +666,7 @@ def get_hosts_to_run_on(oo_cfg, callback_facts, unattended, force, verbose):                      openshift_ansible.set_config(oo_cfg)                      click.echo('Gathering information from hosts...')                      callback_facts, error = openshift_ansible.default_facts(oo_cfg.hosts, verbose) -                    if error: +                    if error or callback_facts is None:                          click.echo("There was a problem fetching the required information. See " \                                     "{} for details.".format(oo_cfg.settings['ansible_log_path']))                          sys.exit(1) @@ -780,42 +780,67 @@ def uninstall(ctx):  @click.command() +@click.option('--latest-minor', '-l', is_flag=True, default=False) +@click.option('--next-major', '-n', is_flag=True, default=False)  @click.pass_context -def upgrade(ctx): +def upgrade(ctx, latest_minor, next_major):      oo_cfg = ctx.obj['oo_cfg']      verbose = ctx.obj['verbose'] +    upgrade_mappings = { +                        '3.0':{ +                               'minor_version' :'3.0', +                               'minor_playbook':'v3_0_minor/upgrade.yml', +                               'major_version' :'3.1', +                               'major_playbook':'v3_0_to_v3_1/upgrade.yml', +                              }, +                        '3.1':{ +                               'minor_version' :'3.1', +                               'minor_playbook':'v3_1_minor/upgrade.yml', +                               'major_playbook':'v3_1_to_v3_2/upgrade.yml', +                               'major_version' :'3.2', +                            } +                       } +      if len(oo_cfg.hosts) == 0:          click.echo("No hosts defined in: %s" % oo_cfg.config_path)          sys.exit(1)      old_variant = oo_cfg.settings['variant']      old_version = oo_cfg.settings['variant_version'] - +    mapping = upgrade_mappings.get(old_version)      message = """          This tool will help you upgrade your existing OpenShift installation.  """      click.echo(message) -    click.echo("Version {} found. Do you want to update to the latest version of {} " \ -               "or migrate to the next major release?".format(old_version, old_version)) -    resp = click.prompt("(1) Update to latest {} (2) Migrate to next relese".format(old_version)) -    if resp == "2": -        # TODO: Make this a lot more flexible -        new_version = "3.1" +    if not (latest_minor or next_major): +        click.echo("Version {} found. Do you want to update to the latest version of {} " \ +                   "or migrate to the next major release?".format(old_version, old_version)) +        response = click.prompt("(1) Update to latest {} " \ +                                "(2) Migrate to next release".format(old_version), +                                type=click.Choice(['1', '2']),) +        if response == "1": +            latest_minor = True +        if response == "2": +            next_major = True + +    if next_major: +        playbook = mapping['major_playbook'] +        new_version = mapping['major_version']          # Update config to reflect the version we're targetting, we'll write          # to disk once ansible completes successfully, not before. +        oo_cfg.settings['variant_version'] = new_version          if oo_cfg.settings['variant'] == 'enterprise':              oo_cfg.settings['variant'] = 'openshift-enterprise' -        version = find_variant(oo_cfg.settings['variant'])[1] -        oo_cfg.settings['variant_version'] = version.name -    else: -        new_version = old_version + +    if latest_minor: +        playbook = mapping['minor_playbook'] +        new_version = mapping['minor_version']      click.echo("Openshift will be upgraded from %s %s to %s %s on the following hosts:\n" % ( -        old_variant, old_version, oo_cfg.settings['variant'], -        oo_cfg.settings['variant_version'])) +        old_variant, old_version, oo_cfg.settings['variant'], new_version))      for host in oo_cfg.hosts:          click.echo("  * %s" % host.connect_to) @@ -826,7 +851,7 @@ def upgrade(ctx):              click.echo("Upgrade cancelled.")              sys.exit(0) -    retcode = openshift_ansible.run_upgrade_playbook(old_version, new_version, verbose) +    retcode = openshift_ansible.run_upgrade_playbook(playbook, verbose)      if retcode > 0:          click.echo("Errors encountered during upgrade, please check %s." %              oo_cfg.settings['ansible_log_path']) @@ -837,8 +862,10 @@ def upgrade(ctx):  @click.command()  @click.option('--force', '-f', is_flag=True, default=False) +@click.option('--gen-inventory', is_flag=True, default=False, +              help="Generate an ansible inventory file and exit.")  @click.pass_context -def install(ctx, force): +def install(ctx, force, gen_inventory):      oo_cfg = ctx.obj['oo_cfg']      verbose = ctx.obj['verbose'] @@ -853,7 +880,7 @@ def install(ctx, force):      click.echo('Gathering information from hosts...')      callback_facts, error = openshift_ansible.default_facts(oo_cfg.hosts,          verbose) -    if error: +    if error or callback_facts is None:          click.echo("There was a problem fetching the required information. " \                     "Please see {} for details.".format(oo_cfg.settings['ansible_log_path']))          sys.exit(1) @@ -861,7 +888,6 @@ def install(ctx, force):      hosts_to_run_on, callback_facts = get_hosts_to_run_on(          oo_cfg, callback_facts, ctx.obj['unattended'], force, verbose) -    click.echo('Writing config to: %s' % oo_cfg.config_path)      # We already verified this is not the case for unattended installs, so this can      # only trigger for live CLI users: @@ -871,7 +897,18 @@ def install(ctx, force):      if len(oo_cfg.calc_missing_facts()) > 0:          confirm_hosts_facts(oo_cfg, callback_facts) +    # Write quick installer config file to disk:      oo_cfg.save_to_disk() +    # Write ansible inventory file to disk: +    inventory_file = openshift_ansible.generate_inventory(hosts_to_run_on) + +    click.echo() +    click.echo('Wrote atomic-openshift-installer config: %s' % oo_cfg.config_path) +    click.echo("Wrote ansible inventory: %s" % inventory_file) +    click.echo() + +    if gen_inventory: +        sys.exit(0)      click.echo('Ready to run installation process.')      message = """ @@ -880,8 +917,8 @@ If changes are needed please edit the config file above and re-run.      if not ctx.obj['unattended']:          confirm_continue(message) -    error = openshift_ansible.run_main_playbook(oo_cfg.hosts, -                                                   hosts_to_run_on, verbose) +    error = openshift_ansible.run_main_playbook(inventory_file, oo_cfg.hosts, +                                                hosts_to_run_on, verbose)      if error:          # The bootstrap script will print out the log location.          message = """ diff --git a/utils/src/ooinstall/oo_config.py b/utils/src/ooinstall/oo_config.py index c9498542f..0f1f5caf7 100644 --- a/utils/src/ooinstall/oo_config.py +++ b/utils/src/ooinstall/oo_config.py @@ -198,7 +198,7 @@ class OOConfig(object):              self.settings['ansible_ssh_user'] = ''          self.settings['ansible_inventory_path'] = \ -            '{}/hosts'.format(self.settings['ansible_inventory_directory']) +            '{}/hosts'.format(os.path.dirname(self.config_path))          # clean up any empty sets          for setting in self.settings.keys(): diff --git a/utils/src/ooinstall/openshift_ansible.py b/utils/src/ooinstall/openshift_ansible.py index 2b95702bf..28b157e8e 100644 --- a/utils/src/ooinstall/openshift_ansible.py +++ b/utils/src/ooinstall/openshift_ansible.py @@ -213,9 +213,8 @@ def default_facts(hosts, verbose=False):      return load_system_facts(inventory_file, os_facts_path, facts_env, verbose) -def run_main_playbook(hosts, hosts_to_run_on, verbose=False): +def run_main_playbook(inventory_file, hosts, hosts_to_run_on, verbose=False):      global CFG -    inventory_file = generate_inventory(hosts_to_run_on)      if len(hosts_to_run_on) != len(hosts):          main_playbook_path = os.path.join(CFG.ansible_playbook_directory,                                            'playbooks/byo/openshift-node/scaleup.yml') @@ -251,18 +250,10 @@ def run_uninstall_playbook(verbose=False):      return run_ansible(playbook, inventory_file, facts_env, verbose) -def run_upgrade_playbook(old_version, new_version, verbose=False): -    # TODO: do not hardcode the upgrade playbook, add ability to select the -    # right playbook depending on the type of upgrade. -    old_version = old_version.replace('.', '_') -    new_version = old_version.replace('.', '_') -    if old_version == new_version: -        playbook = os.path.join(CFG.settings['ansible_playbook_directory'], -            'playbooks/byo/openshift-cluster/upgrades/v{}_minor/upgrade.yml'.format(new_version)) -    else: -        playbook = os.path.join(CFG.settings['ansible_playbook_directory'], -            'playbooks/byo/openshift-cluster/upgrades/v{}_to_v{}/upgrade.yml'.format(old_version, -                                                                                     new_version)) +def run_upgrade_playbook(playbook, verbose=False): +    playbook = os.path.join(CFG.settings['ansible_playbook_directory'], +            'playbooks/byo/openshift-cluster/upgrades/{}'.format(playbook)) +      # TODO: Upgrade inventory for upgrade?      inventory_file = generate_inventory(CFG.hosts)      facts_env = os.environ.copy() diff --git a/utils/src/ooinstall/variants.py b/utils/src/ooinstall/variants.py index 571025543..9d98379bb 100644 --- a/utils/src/ooinstall/variants.py +++ b/utils/src/ooinstall/variants.py @@ -36,6 +36,7 @@ class Variant(object):  # WARNING: Keep the versions ordered, most recent last:  OSE = Variant('openshift-enterprise', 'OpenShift Enterprise',      [ +        Version('3.2', 'openshift-enterprise'),          Version('3.1', 'openshift-enterprise'),          Version('3.0', 'enterprise')      ] @@ -43,6 +44,7 @@ OSE = Variant('openshift-enterprise', 'OpenShift Enterprise',  AEP = Variant('atomic-enterprise', 'Atomic Enterprise Platform',      [ +        Version('3.2', 'atomic-enterprise'),          Version('3.1', 'atomic-enterprise')      ]  ) @@ -74,4 +76,3 @@ def get_variant_version_combos():          for ver in variant.versions:              combos.append((variant, ver))      return combos - diff --git a/utils/test/cli_installer_tests.py b/utils/test/cli_installer_tests.py index 6ba5ec1eb..524df08c4 100644 --- a/utils/test/cli_installer_tests.py +++ b/utils/test/cli_installer_tests.py @@ -1,6 +1,6 @@  # TODO: Temporarily disabled due to importing old code into openshift-ansible  # repo. We will work on these over time. -# pylint: disable=bad-continuation,missing-docstring,no-self-use,invalid-name +# pylint: disable=bad-continuation,missing-docstring,no-self-use,invalid-name,too-many-lines  import copy  import os @@ -403,7 +403,7 @@ class UnattendedCliTests(OOCliFixture):          self.assert_result(result, 0)          load_facts_args = load_facts_mock.call_args[0] -        self.assertEquals(os.path.join(self.work_dir, ".ansible/hosts"), +        self.assertEquals(os.path.join(self.work_dir, "hosts"),              load_facts_args[0])          self.assertEquals(os.path.join(self.work_dir,              "playbooks/byo/openshift_facts.yml"), load_facts_args[1]) @@ -417,8 +417,8 @@ class UnattendedCliTests(OOCliFixture):              env_vars['ANSIBLE_CONFIG'] == cli.DEFAULT_ANSIBLE_CONFIG)          # Make sure we ran on the expected masters and nodes: -        hosts = run_playbook_mock.call_args[0][0] -        hosts_to_run_on = run_playbook_mock.call_args[0][1] +        hosts = run_playbook_mock.call_args[0][1] +        hosts_to_run_on = run_playbook_mock.call_args[0][2]          self.assertEquals(3, len(hosts))          self.assertEquals(3, len(hosts_to_run_on)) @@ -441,7 +441,7 @@ class UnattendedCliTests(OOCliFixture):          # Check the inventory file looks as we would expect:          inventory = ConfigParser.ConfigParser(allow_no_value=True) -        inventory.read(os.path.join(self.work_dir, '.ansible/hosts')) +        inventory.read(os.path.join(self.work_dir, 'hosts'))          self.assertEquals('bob',              inventory.get('OSEv3:vars', 'ansible_ssh_user'))          self.assertEquals('openshift-enterprise', @@ -480,11 +480,11 @@ class UnattendedCliTests(OOCliFixture):          self.assertEquals('openshift-enterprise', written_config['variant'])          # We didn't specify a version so the latest should have been assumed,          # and written to disk: -        self.assertEquals('3.1', written_config['variant_version']) +        self.assertEquals('3.2', written_config['variant_version'])          # Make sure the correct value was passed to ansible:          inventory = ConfigParser.ConfigParser(allow_no_value=True) -        inventory.read(os.path.join(self.work_dir, '.ansible/hosts')) +        inventory.read(os.path.join(self.work_dir, 'hosts'))          self.assertEquals('openshift-enterprise',              inventory.get('OSEv3:vars', 'deployment_type')) @@ -512,7 +512,7 @@ class UnattendedCliTests(OOCliFixture):          self.assertEquals('3.0', written_config['variant_version'])          inventory = ConfigParser.ConfigParser(allow_no_value=True) -        inventory.read(os.path.join(self.work_dir, '.ansible/hosts')) +        inventory.read(os.path.join(self.work_dir, 'hosts'))          self.assertEquals('enterprise',              inventory.get('OSEv3:vars', 'deployment_type')) @@ -625,8 +625,8 @@ class UnattendedCliTests(OOCliFixture):          self.assert_result(result, 0)          # Make sure we ran on the expected masters and nodes: -        hosts = run_playbook_mock.call_args[0][0] -        hosts_to_run_on = run_playbook_mock.call_args[0][1] +        hosts = run_playbook_mock.call_args[0][1] +        hosts_to_run_on = run_playbook_mock.call_args[0][2]          self.assertEquals(6, len(hosts))          self.assertEquals(6, len(hosts_to_run_on)) @@ -695,8 +695,8 @@ class UnattendedCliTests(OOCliFixture):          self.assert_result(result, 0)          # Make sure we ran on the expected masters and nodes: -        hosts = run_playbook_mock.call_args[0][0] -        hosts_to_run_on = run_playbook_mock.call_args[0][1] +        hosts = run_playbook_mock.call_args[0][1] +        hosts_to_run_on = run_playbook_mock.call_args[0][2]          self.assertEquals(6, len(hosts))          self.assertEquals(6, len(hosts_to_run_on)) @@ -733,13 +733,13 @@ class AttendedCliTests(OOCliFixture):          self._verify_config_hosts(written_config, 3)          inventory = ConfigParser.ConfigParser(allow_no_value=True) -        inventory.read(os.path.join(self.work_dir, '.ansible/hosts')) -        self.assertEquals('False', -            inventory.get('nodes', '10.0.0.1  openshift_schedulable')) -        self.assertEquals(None, -            inventory.get('nodes', '10.0.0.2')) -        self.assertEquals(None, -            inventory.get('nodes', '10.0.0.3')) +        inventory.read(os.path.join(self.work_dir, 'hosts')) +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.1', +                                 'openshift_schedulable=False') +        self.assert_inventory_host_var_unset(inventory, 'nodes', '10.0.0.2', +                                 'openshift_schedulable') +        self.assert_inventory_host_var_unset(inventory, 'nodes', '10.0.0.3', +                                 'openshift_schedulable')      # interactive with config file and some installed some uninstalled hosts      @patch('ooinstall.openshift_ansible.run_main_playbook') @@ -851,15 +851,15 @@ class AttendedCliTests(OOCliFixture):          self._verify_config_hosts(written_config, 6)          inventory = ConfigParser.ConfigParser(allow_no_value=True) -        inventory.read(os.path.join(self.work_dir, '.ansible/hosts')) -        self.assertEquals('False', -            inventory.get('nodes', '10.0.0.1  openshift_schedulable')) -        self.assertEquals('False', -            inventory.get('nodes', '10.0.0.2  openshift_schedulable')) -        self.assertEquals('False', -            inventory.get('nodes', '10.0.0.3  openshift_schedulable')) -        self.assertEquals(None, -            inventory.get('nodes', '10.0.0.4')) +        inventory.read(os.path.join(self.work_dir, 'hosts')) +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.1', +                                       'openshift_schedulable=False') +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.2', +                                       'openshift_schedulable=False') +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.3', +                                       'openshift_schedulable=False') +        self.assert_inventory_host_var_unset(inventory, 'nodes', '10.0.0.4', +                                             'openshift_schedulable')          self.assertTrue(inventory.has_section('etcd'))          self.assertEquals(3, len(inventory.items('etcd'))) @@ -892,13 +892,50 @@ class AttendedCliTests(OOCliFixture):          self._verify_config_hosts(written_config, 5)          inventory = ConfigParser.ConfigParser(allow_no_value=True) -        inventory.read(os.path.join(self.work_dir, '.ansible/hosts')) -        self.assertEquals('True', -            inventory.get('nodes', '10.0.0.1  openshift_schedulable')) -        self.assertEquals('True', -            inventory.get('nodes', '10.0.0.2  openshift_schedulable')) -        self.assertEquals('True', -            inventory.get('nodes', '10.0.0.3  openshift_schedulable')) +        inventory.read(os.path.join(self.work_dir, 'hosts')) +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.1', +                                       'openshift_schedulable=True') +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.2', +                                       'openshift_schedulable=True') +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.3', +                                       'openshift_schedulable=True') + +    # Checks the inventory (as a ConfigParser) for the given host, host +    # variable, and expected value. +    def assert_inventory_host_var(self, inventory, section, host, variable): +        # Config parser splits on the first "=", so we end up with: +        #   'hostname key1' -> 'val1 key2=val2 key3=val3' +        # +        # Convert to something easier to test: +        for (a, b) in inventory.items(section): +            full_line = "%s=%s" % (a, b) +            tokens = full_line.split() +            if tokens[0] == host: +                found = False +                for token in tokens: +                    if token == variable: +                        found = True +                        continue +                self.assertTrue("Unable to find %s in line: %s" % +                                (variable, full_line), found) +                return +        self.fail("unable to find host %s in inventory" % host) + +    def assert_inventory_host_var_unset(self, inventory, section, host, variable): +        # Config parser splits on the first "=", so we end up with: +        #   'hostname key1' -> 'val1 key2=val2 key3=val3' +        # +        # Convert to something easier to test: +        for (a, b) in inventory.items(section): +            full_line = "%s=%s" % (a, b) +            tokens = full_line.split() +            if tokens[0] == host: +                self.assertFalse(("%s=" % variable) in full_line, +                                 msg='%s host variable was set: %s' % +                                 (variable, full_line)) +                return +        self.fail("unable to find host %s in inventory" % host) +      #interactive multimaster: attempting to use a master as the load balancer should fail:      @patch('ooinstall.openshift_ansible.run_main_playbook') @@ -946,9 +983,9 @@ class AttendedCliTests(OOCliFixture):          self._verify_config_hosts(written_config, 1)          inventory = ConfigParser.ConfigParser(allow_no_value=True) -        inventory.read(os.path.join(self.work_dir, '.ansible/hosts')) -        self.assertEquals('True', -            inventory.get('nodes', '10.0.0.1  openshift_schedulable')) +        inventory.read(os.path.join(self.work_dir, 'hosts')) +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.1', +                                       'openshift_schedulable=True')      #interactive 3.0 install confirm no HA hints      @patch('ooinstall.openshift_ansible.run_main_playbook') @@ -960,15 +997,53 @@ class AttendedCliTests(OOCliFixture):          cli_input = build_input(hosts=[              ('10.0.0.1', True, False)],                                        ssh_user='root', -                                      variant_num=2, +                                      variant_num=3,                                        confirm_facts='y')          self.cli_args.append("install")          result = self.runner.invoke(cli.cli, self.cli_args,              input=cli_input)          self.assert_result(result, 0) +        print result.output          self.assertTrue("NOTE: Add a total of 3 or more Masters to perform an HA installation."              not in result.output) +    @patch('ooinstall.openshift_ansible.run_main_playbook') +    @patch('ooinstall.openshift_ansible.load_system_facts') +    def test_gen_inventory(self, load_facts_mock, run_playbook_mock): +        load_facts_mock.return_value = (MOCK_FACTS, 0) +        run_playbook_mock.return_value = 0 + +        cli_input = build_input(hosts=[ +            ('10.0.0.1', True, False), +            ('10.0.0.2', False, False), +            ('10.0.0.3', False, False)], +                                ssh_user='root', +                                variant_num=1, +                                confirm_facts='y') +        self.cli_args.append("install") +        self.cli_args.append("--gen-inventory") +        result = self.runner.invoke(cli.cli, self.cli_args, +            input=cli_input) +        self.assert_result(result, 0) + +        self._verify_load_facts(load_facts_mock) + +        # Make sure run playbook wasn't called: +        self.assertEquals(0, len(run_playbook_mock.mock_calls)) + +        written_config = read_yaml(self.config_file) +        self._verify_config_hosts(written_config, 3) + +        inventory = ConfigParser.ConfigParser(allow_no_value=True) +        inventory.read(os.path.join(self.work_dir, 'hosts')) +        self.assert_inventory_host_var(inventory, 'nodes', '10.0.0.1', +                                 'openshift_schedulable=False') +        self.assert_inventory_host_var_unset(inventory, 'nodes', '10.0.0.2', +                                 'openshift_schedulable') +        self.assert_inventory_host_var_unset(inventory, 'nodes', '10.0.0.3', +                                 'openshift_schedulable') + +  # TODO: test with config file, attended add node  # TODO: test with config file, attended new node already in config file  # TODO: test with config file, attended new node already in config file, plus manually added nodes diff --git a/utils/test/fixture.py b/utils/test/fixture.py index d6222dfaa..1657d8f46 100644 --- a/utils/test/fixture.py +++ b/utils/test/fixture.py @@ -68,7 +68,7 @@ class OOCliFixture(OOInstallFixture):      def _verify_load_facts(self, load_facts_mock):          """ Check that we ran load facts with expected inputs. """          load_facts_args = load_facts_mock.call_args[0] -        self.assertEquals(os.path.join(self.work_dir, ".ansible/hosts"), +        self.assertEquals(os.path.join(self.work_dir, "hosts"),                            load_facts_args[0])          self.assertEquals(os.path.join(self.work_dir,                                         "playbooks/byo/openshift_facts.yml"), @@ -81,8 +81,8 @@ class OOCliFixture(OOInstallFixture):      def _verify_run_playbook(self, run_playbook_mock, exp_hosts_len, exp_hosts_to_run_on_len):          """ Check that we ran playbook with expected inputs. """ -        hosts = run_playbook_mock.call_args[0][0] -        hosts_to_run_on = run_playbook_mock.call_args[0][1] +        hosts = run_playbook_mock.call_args[0][1] +        hosts_to_run_on = run_playbook_mock.call_args[0][2]          self.assertEquals(exp_hosts_len, len(hosts))          self.assertEquals(exp_hosts_to_run_on_len, len(hosts_to_run_on)) @@ -133,8 +133,8 @@ class OOCliFixture(OOInstallFixture):          self._verify_run_playbook(run_playbook_mock, exp_hosts_len, exp_hosts_to_run_on_len)          # Make sure we ran on the expected masters and nodes: -        hosts = run_playbook_mock.call_args[0][0] -        hosts_to_run_on = run_playbook_mock.call_args[0][1] +        hosts = run_playbook_mock.call_args[0][1] +        hosts_to_run_on = run_playbook_mock.call_args[0][2]          self.assertEquals(exp_hosts_len, len(hosts))          self.assertEquals(exp_hosts_to_run_on_len, len(hosts_to_run_on)) | 
