Merge pull request #535 from bogdando/bastion_inventory

Options for bastion, SSH config, static inventory autogeneration

16 files changed, 312 insertions, 15 deletions

diff --git a/playbooks/provisioning/openstack/README.md b/playbooks/provisioning/openstack/README.md
index 1ff586b49..6b9e5a3a9 100644
--- a/playbooks/provisioning/openstack/README.md
+++ b/playbooks/provisioning/openstack/README.md
@@ -40,7 +40,7 @@ Alternatively you can install directly from github:
       -p openshift-ansible-contrib/roles
 
 Notes:
-* This assumes we're in the directory that contains the clonned 
+* This assumes we're in the directory that contains the clonned
 openshift-ansible-contrib repo in its root path.
 * When trying to install a different version, the previous one must be removed first
 (`infra-ansible` directory from [roles](https://github.com/openshift/openshift-ansible-contrib/tree/master/roles)).
@@ -177,16 +177,30 @@ variables for the `inventory/group_vars/OSEv3.yml`, `all.yml`:
     origin_release: 1.5.1
     openshift_deployment_type: "{{ deployment_type }}"
 
-### Configure static inventory
+### Configure static inventory and access via a bastion node
 
 Example inventory variables:
 
+    openstack_use_bastion: true
+    bastion_ingress_cidr: "{{openstack_subnet_prefix}}.0/24"
     openstack_private_ssh_key: ~/.ssh/openshift
     openstack_inventory: static
     openstack_inventory_path: ../../../../inventory
+    openstack_ssh_config_path: /tmp/ssh.config.openshift.ansible.openshift.example.com
 
+The `openstack_subnet_prefix` is the openstack private network for your cluster.
+And the `bastion_ingress_cidr` defines accepted range for SSH connections to nodes
+additionally to the `ssh_ingress_cidr`` (see the security notes above).
 
-In this guide, the latter points to the current directory, where you run ansible commands
+The SSH config will be stored on the ansible control node by the
+gitven path. Ansible uses it automatically. To access the cluster nodes with
+that ssh config, use the `-F` prefix, f.e.:
+
+    ssh -F /tmp/ssh.config.openshift.ansible.openshift.example.com master-0.openshift.example.com echo OK
+
+Note, relative paths will not work for the `openstack_ssh_config_path`, but it
+works for the `openstack_private_ssh_key` and `openstack_inventory_path`. In this
+guide, the latter points to the current directory, where you run ansible commands
 from.
 
 To verify nodes connectivity, use the command:
@@ -194,7 +208,7 @@ To verify nodes connectivity, use the command:
     ansible -v -i inventory/hosts -m ping all
 
 If something is broken, double-check the inventory variables, paths and the
-generated `<openstack_inventory_path>/hosts` file.
+generated `<openstack_inventory_path>/hosts` and `openstack_ssh_config_path` files.
 
 The `inventory: dynamic` can be used instead to access cluster nodes directly via
 floating IPs. In this mode you can not use a bastion node and should specify
@@ -213,6 +227,15 @@ this is how you stat the provisioning process from your ansible control node:
 Note, here you start with an empty inventory. The static inventory will be populated
 with data so you can omit providing additional arguments for future ansible commands.
 
+If bastion enabled, the generates SSH config must be applied for ansible.
+Otherwise, it is auto included by the previous step. In order to execute it
+as a separate playbook, use the following command:
+
+    ansible-playbook openshift-ansible-contrib/playbooks/provisioning/openstack/post-provision-openstack.yml
+
+The first infra node then becomes a bastion node as well and proxies access
+for future ansible commands. The post-provision step also configures Satellite,
+if requested, and DNS server, and ensures other OpenShift requirements to be met.
 
 ### Install OpenShift
 
diff --git a/playbooks/provisioning/openstack/openstack_dns_records.yml b/playbooks/provisioning/openstack/openstack_dns_records.yml
index b5f0840c5..980221ed6 100644
--- a/playbooks/provisioning/openstack/openstack_dns_records.yml
+++ b/playbooks/provisioning/openstack/openstack_dns_records.yml
@@ -36,11 +36,13 @@
   set_fact:
     public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': hostvars[item]['ansible_hostname'], 'ip': hostvars[item]['public_v4'] } ] }}"
   with_items: "{{ groups['cluster_hosts'] }}"
+  when: hostvars[item]['public_v4'] is defined
 
 - name: "Add wildcard records to the public A records"
   set_fact:
     public_records: "{{ public_records | default([]) + [ { 'type': 'A', 'hostname': '*.' + openshift_app_domain, 'ip': hostvars[item]['public_v4'] } ] }}"
   with_items: "{{ groups['infra_hosts'] }}"
+  when: hostvars[item]['public_v4'] is defined
 
 - name: "Set the public DNS server details to use the external value (if provided)"
   set_fact:
diff --git a/playbooks/provisioning/openstack/post-provision-openstack.yml b/playbooks/provisioning/openstack/post-provision-openstack.yml
index a807c4d2f..c7df74a87 100644
--- a/playbooks/provisioning/openstack/post-provision-openstack.yml
+++ b/playbooks/provisioning/openstack/post-provision-openstack.yml
@@ -4,7 +4,11 @@
   become: False
   gather_facts: False
   tasks:
-    - wait_for_connection:
+    - when: not openstack_use_bastion|default(False)|bool
+      wait_for_connection:
+    - when: openstack_use_bastion|default(False)|bool
+      delegate_to: bastion
+      wait_for_connection:
 
 - hosts: cluster_hosts
   gather_facts: True
diff --git a/playbooks/provisioning/openstack/provision-openstack.yml b/playbooks/provisioning/openstack/provision-openstack.yml
index 0cac37aaf..6ec944d56 100644
--- a/playbooks/provisioning/openstack/provision-openstack.yml
+++ b/playbooks/provisioning/openstack/provision-openstack.yml
@@ -12,13 +12,20 @@
       when: openstack_inventory|default('static') == 'static'
       inventory_path: "{{ openstack_inventory_path|default(inventory_dir) }}"
       private_ssh_key: "{{ openstack_private_ssh_key|default('~/.ssh/id_rsa') }}"
+      ssh_config_path: "{{ openstack_ssh_config_path|default('/tmp/ssh.config.openshift.ansible' + '.' + stack_name) }}"
+      ssh_user: "{{ ansible_user }}"
 
-- name: Refresh Server inventory
+- name: Refresh Server inventory or exit to apply SSH config
   hosts: localhost
   connection: local
   become: False
   gather_facts: False
   tasks:
-    - meta: refresh_inventory
+    - name: Exit to apply SSH config for a bastion
+      meta: end_play
+      when: openstack_use_bastion|default(False)|bool
+    - name: Refresh Server inventory
+      meta: refresh_inventory
 
 - include: post-provision-openstack.yml
+  when: not openstack_use_bastion|default(False)|bool
diff --git a/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml b/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml
index 9eb36ab13..6d07f9b56 100644
--- a/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml
+++ b/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml
@@ -69,5 +69,12 @@ ansible_user: openshift
 # # The path to checkpoint the static inventory from the in-memory one
 #openstack_inventory_path: ../../../../inventory
 
+# # Use bastion node to access cluster nodes (Defaults to False).
+# # Requires a static inventory.
+#openstack_use_bastion: False
+#bastion_ingress_cidr: "{{openstack_subnet_prefix}}.0/24"
+#
 # # The Nova key-pair's private SSH key to access inventory nodes
 #openstack_private_ssh_key: ~/.ssh/openshift
+# # The path for the SSH config to access all nodes
+#openstack_ssh_config_path: /tmp/ssh.config.openshift.ansible.{{ env_id }}.{{ public_dns_domain }}
diff --git a/playbooks/provisioning/openstack/stack_params.yaml b/playbooks/provisioning/openstack/stack_params.yaml
index 9c0b09b45..c3a42ab06 100644
--- a/playbooks/provisioning/openstack/stack_params.yaml
+++ b/playbooks/provisioning/openstack/stack_params.yaml
@@ -21,3 +21,4 @@ master_volume_size: "{{ docker_volume_size }}"
 app_volume_size: "{{ docker_volume_size }}"
 infra_volume_size: "{{ docker_volume_size }}"
 nodes_to_remove: "{{ openstack_nodes_to_remove | default([]) |  to_yaml }}"
+use_bastion: "{{ openstack_use_bastion|default(False) }}"
diff --git a/roles/openstack-stack/defaults/main.yml b/roles/openstack-stack/defaults/main.yml
index 4831d6bc4..803a96389 100644
--- a/roles/openstack-stack/defaults/main.yml
+++ b/roles/openstack-stack/defaults/main.yml
@@ -4,6 +4,7 @@ ssh_ingress_cidr: 0.0.0.0/0
 node_ingress_cidr: 0.0.0.0/0
 master_ingress_cidr: 0.0.0.0/0
 lb_ingress_cidr: 0.0.0.0/0
+bastion_ingress_cidr: 0.0.0.0/0
 num_etcd: 0
 num_masters: 1
 num_nodes: 1
@@ -11,3 +12,4 @@ num_dns: 1
 num_infra: 1
 nodes_to_remove: []
 etcd_volume_size: 2
+use_bastion: False
diff --git a/roles/openstack-stack/tasks/main.yml b/roles/openstack-stack/tasks/main.yml
index a53e6350b..9b4855294 100644
--- a/roles/openstack-stack/tasks/main.yml
+++ b/roles/openstack-stack/tasks/main.yml
@@ -8,7 +8,6 @@
 - name: set template paths
   set_fact:
     stack_template_path: "{{ stack_template_pre.path }}/stack.yaml"
-    server_template_path: "{{ stack_template_pre.path }}/server.yaml"
     user_data_template_path: "{{ stack_template_pre.path }}/user-data"
 
 - name: generate HOT stack template from jinja2 template
@@ -19,7 +18,13 @@
 - name: generate HOT server template from jinja2 template
   template:
     src: heat_stack_server.yaml.j2
-    dest: "{{ server_template_path }}"
+    dest: "{{ stack_template_pre.path }}/server.yaml"
+
+- name: generate HOT server w/o floating IPs template from jinja2 template
+  template:
+    src: heat_stack_server_nofloating.yaml.j2
+    dest: "{{ stack_template_pre.path }}/server_nofloating.yaml"
+  when: use_bastion|bool
 
 - name: generate user_data from jinja2 template
   template:
diff --git a/roles/openstack-stack/templates/heat_stack.yaml.j2 b/roles/openstack-stack/templates/heat_stack.yaml.j2
index 54941db06..524f466ff 100644
--- a/roles/openstack-stack/templates/heat_stack.yaml.j2
+++ b/roles/openstack-stack/templates/heat_stack.yaml.j2
@@ -156,6 +156,13 @@ resources:
           port_range_min: 22
           port_range_max: 22
           remote_ip_prefix: {{ ssh_ingress_cidr }}
+{% if use_bastion|bool %}
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 22
+          port_range_max: 22
+          remote_ip_prefix: {{ bastion_ingress_cidr }}
+{% endif %}
         - direction: ingress
           protocol: icmp
           remote_ip_prefix: {{ ssh_ingress_cidr }}
@@ -458,7 +465,11 @@ resources:
     properties:
       count: {{ num_etcd }}
       resource_def:
+{% if use_bastion|bool %}
+        type: server_nofloating.yaml
+{% else %}
         type: server.yaml
+{% endif %}
         properties:
           name:
             str_replace:
@@ -483,7 +494,9 @@ resources:
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}etcd-secgrp{% endif %} }
             - { get_resource: common-secgrp }
+{% if not use_bastion|bool %}
           floating_network: {{ external_network }}
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -540,7 +553,11 @@ resources:
     properties:
       count: {{ num_masters }}
       resource_def:
+{% if use_bastion|bool %}
+        type: server_nofloating.yaml
+{% else %}
         type: server.yaml
+{% endif %}
         properties:
           name:
             str_replace:
@@ -573,7 +590,9 @@ resources:
 {% endif %}
 {% endif %}
             - { get_resource: common-secgrp }
+{% if not use_bastion|bool %}
           floating_network: {{ external_network }}
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
@@ -590,7 +609,11 @@ resources:
       removal_policies:
       - resource_list: {{ nodes_to_remove }}
       resource_def:
+{% if use_bastion|bool %}
+        type: server_nofloating.yaml
+{% else %}
         type: server.yaml
+{% endif %}
         properties:
           name:
             str_replace:
@@ -621,7 +644,9 @@ resources:
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}node-secgrp{% endif %} }
             - { get_resource: common-secgrp }
+{% if not use_bastion|bool %}
           floating_network: {{ external_network }}
+{% endif %}
           net_name:
             str_replace:
               template: openshift-ansible-cluster_id-net
diff --git a/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2 b/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2
new file mode 100644
index 000000000..792a8b90c
--- /dev/null
+++ b/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2
@@ -0,0 +1,149 @@
+heat_template_version: 2016-10-14
+
+description: OpenShift cluster server w/o floating IP
+
+parameters:
+
+  name:
+    type: string
+    label: Name
+    description: Name
+
+  group:
+    type: string
+    label: Host Group
+    description: The Primary Ansible Host Group
+    default: host
+
+  cluster_env:
+    type: string
+    label: Cluster environment
+    description: Environment of the cluster
+
+  cluster_id:
+    type: string
+    label: Cluster ID
+    description: Identifier of the cluster
+
+  type:
+    type: string
+    label: Type
+    description: Type master or node
+
+  subtype:
+    type: string
+    label: Sub-type
+    description: Sub-type compute or infra for nodes, default otherwise
+    default: default
+
+  key_name:
+    type: string
+    label: Key name
+    description: Key name of keypair
+
+  image:
+    type: string
+    label: Image
+    description: Name of the image
+
+  flavor:
+    type: string
+    label: Flavor
+    description: Name of the flavor
+
+  net:
+    type: string
+    label: Net ID
+    description: Net resource
+
+  net_name:
+    type: string
+    label: Net name
+    description: Net name
+
+  subnet:
+    type: string
+    label: Subnet ID
+    description: Subnet resource
+
+  secgrp:
+    type: comma_delimited_list
+    label: Security groups
+    description: Security group resources
+
+  availability_zone:
+    type: string
+    description: The Availability Zone to launch the instance.
+    default: nova
+
+  volume_size:
+    type: number
+    description: Size of the volume to be created.
+    default: 1
+    constraints:
+      - range: { min: 1, max: 1024 }
+        description: must be between 1 and 1024 Gb.
+
+  node_labels:
+    type: json
+    description: OpenShift Node Labels
+    default: {"region": "default" }
+
+outputs:
+
+  name:
+    description: Name of the server
+    value: { get_attr: [ server_nofloating, name ] }
+
+  private_ip:
+    description: Private IP of the server
+    value:
+      get_attr:
+        - server_nofloating
+        - addresses
+        - { get_param: net_name }
+        - 0
+        - addr
+
+resources:
+
+  server_nofloating:
+    type: OS::Nova::Server
+    properties:
+      name:      { get_param: name }
+      key_name:  { get_param: key_name }
+      image:     { get_param: image }
+      flavor:    { get_param: flavor }
+      networks:
+        - port:  { get_resource: port }
+      user_data:
+        get_file: user-data
+      user_data_format: RAW
+      metadata:
+        group: { get_param: group }
+        environment: { get_param: cluster_env }
+        clusterid: { get_param: cluster_id }
+        host-type: { get_param: type }
+        sub-host-type:    { get_param: subtype }
+        node_labels: { get_param: node_labels }
+
+  port:
+    type: OS::Neutron::Port
+    properties:
+      network: { get_param: net }
+      fixed_ips:
+        - subnet: { get_param: subnet }
+      security_groups: { get_param: secgrp }
+
+  cinder_volume:
+    type: OS::Cinder::Volume
+    properties:
+      size: { get_param: volume_size }
+      availability_zone: { get_param: availability_zone }
+
+  volume_attachment:
+    type: OS::Cinder::VolumeAttachment
+    properties:
+      volume_id: { get_resource: cinder_volume }
+      instance_uuid: { get_resource: server_nofloating }
+      mountpoint: /dev/sdb
diff --git a/roles/static_inventory/defaults/main.yml b/roles/static_inventory/defaults/main.yml
index 315965cde..63de45646 100644
--- a/roles/static_inventory/defaults/main.yml
+++ b/roles/static_inventory/defaults/main.yml
@@ -4,5 +4,18 @@ refresh_inventory: True
 inventory: static
 inventory_path: ~/openstack-inventory
 
+# Either to configure bastion
+use_bastion: true
+
+# SSH user/key/options to access hosts via bastion
+ssh_user: openshift
+ssh_options: >-
+  -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
+  -o ConnectTimeout=90 -o ControlMaster=auto -o ControlPersist=270s
+  -o ServerAliveInterval=30 -o GSSAPIAuthentication=no
+
 # SSH key to access nodes
 private_ssh_key: ~/.ssh/openshift
+
+# The patch to store the generated config to access bastion/hosts
+ssh_config_path: /tmp/ssh.config.ansible
diff --git a/roles/static_inventory/tasks/main.yml b/roles/static_inventory/tasks/main.yml
index 15c81690e..b58866017 100644
--- a/roles/static_inventory/tasks/main.yml
+++ b/roles/static_inventory/tasks/main.yml
@@ -4,3 +4,7 @@
 
 - name: Checkpoint in-memory data into a static inventory
   include: checkpoint.yml
+
+- name: Generate SSH config for accessing hosts via bastion
+  include: sshconfig.yml
+  when: use_bastion|bool
diff --git a/roles/static_inventory/tasks/openstack.yml b/roles/static_inventory/tasks/openstack.yml
index a25502835..499adf08c 100644
--- a/roles/static_inventory/tasks/openstack.yml
+++ b/roles/static_inventory/tasks/openstack.yml
@@ -16,6 +16,7 @@
 
     - name: set_fact for openstack inventory nodes
       set_fact:
+        registered_bastion_nodes: "{{ (registered_nodes_output.stdout | from_json) | json_query(q) }}"
         registered_nodes_floating: "{{ (registered_nodes_output.stdout | from_json) | json_query(q2) }}"
       vars:
         q: "[] | [?metadata.group=='infra.{{stack_name}}']"
@@ -24,24 +25,41 @@
         - refresh_inventory|bool
 
     - name: Add cluster nodes w/o floating IPs to inventory
-      with_items: "{{ registered_nodes }}"
-      when: not item in registered_nodes_floating
+      with_items: "{{ registered_nodes|difference(registered_nodes_floating) }}"
       add_host:
         name: '{{ item.name }}'
         groups: '{{ item.metadata.group }}'
-        ansible_host: '{{ item.private_v4 }}'
+        ansible_host: "{% if use_bastion|bool %}{{ item.name }}{% else %}{{ item.private_v4 }}{% endif %}"
         ansible_fqdn: '{{ item.name }}'
+        ansible_user: '{{ ssh_user }}'
         ansible_private_key_file: '{{ private_ssh_key }}'
+        ansible_ssh_extra_args: '-F {{ ssh_config_path }}'
         private_v4: '{{ item.private_v4 }}'
 
     - name: Add cluster nodes with floating IPs to inventory
       with_items: "{{ registered_nodes_floating }}"
-      when: item in registered_nodes_floating
       add_host:
         name: '{{ item.name }}'
         groups: '{{ item.metadata.group }}'
-        ansible_host: '{{ item.public_v4 }}'
+        ansible_host: "{% if use_bastion|bool %}{{ item.name }}{% else %}{{ item.public_v4 }}{% endif %}"
         ansible_fqdn: '{{ item.name }}'
+        ansible_user: '{{ ssh_user }}'
         ansible_private_key_file: '{{ private_ssh_key }}'
+        ansible_ssh_extra_args: '-F {{ ssh_config_path }}'
         private_v4: '{{ item.private_v4 }}'
         public_v4: '{{ item.public_v4 }}'
+
+    - name: Add bastion node to inventory
+      add_host:
+        name: bastion
+        groups: bastions
+        ansible_host: '{{ registered_bastion_nodes[0].public_v4 }}'
+        ansible_fqdn: '{{ registered_bastion_nodes[0].name }}'
+        ansible_user: '{{ ssh_user }}'
+        ansible_private_key_file: '{{ private_ssh_key }}'
+        ansible_ssh_extra_args: '-F {{ ssh_config_path }}'
+        private_v4: '{{ registered_bastion_nodes[0].private_v4 }}'
+        public_v4: '{{ registered_bastion_nodes[0].public_v4 }}'
+      when:
+        - registered_bastion_nodes is defined
+        - use_bastion|bool
diff --git a/roles/static_inventory/tasks/sshconfig.yml b/roles/static_inventory/tasks/sshconfig.yml
new file mode 100644
index 000000000..7119fe6ff
--- /dev/null
+++ b/roles/static_inventory/tasks/sshconfig.yml
@@ -0,0 +1,13 @@
+---
+- name: set ssh proxy command prefix for accessing nodes via bastion
+  set_fact:
+    ssh_proxy_command: >-
+      ssh {{ ssh_options }}
+      -i {{ private_ssh_key }}
+      {{ ssh_user }}@{{ hostvars['bastion'].ansible_host }}
+
+- name: regenerate ssh config
+  template:
+    src: openstack_ssh_config.j2
+    dest: "{{ ssh_config_path }}"
+    mode: 0644
diff --git a/roles/static_inventory/templates/inventory.j2 b/roles/static_inventory/templates/inventory.j2
index 464726a0b..24dc9d4a8 100644
--- a/roles/static_inventory/templates/inventory.j2
+++ b/roles/static_inventory/templates/inventory.j2
@@ -10,9 +10,12 @@
 %} private_v4={{ hostvars[host]['private_v4'] }}{% endif %}
 {% if 'public_v4' in hostvars[host]
 %} public_v4={{ hostvars[host]['public_v4'] }}{% endif %}
+{% if 'ansible_user' in hostvars[host]
+%} ansible_user={{ hostvars[host]['ansible_user'] }}{% endif %}
 {% if 'ansible_private_key_file' in hostvars[host]
 %} ansible_private_key_file={{ hostvars[host]['ansible_private_key_file'] }}{% endif %}
- openshift_hostname={{ host }}
+{% if use_bastion|bool and 'ansible_ssh_extra_args' in hostvars[host]
+%} ansible_ssh_extra_args={{ hostvars[host]['ansible_ssh_extra_args']|quote }}{% endif %} openshift_hostname={{ host }}
 
 {% endif %}
 {% endfor %}
diff --git a/roles/static_inventory/templates/openstack_ssh_config.j2 b/roles/static_inventory/templates/openstack_ssh_config.j2
new file mode 100644
index 000000000..ad5d1253a
--- /dev/null
+++ b/roles/static_inventory/templates/openstack_ssh_config.j2
@@ -0,0 +1,21 @@
+Host *
+    IdentitiesOnly yes
+
+Host bastion
+    Hostname {{ hostvars['bastion'].ansible_host }}
+    IdentityFile {{ hostvars['bastion'].ansible_private_key_file }}
+    User {{ ssh_user }}
+    StrictHostKeyChecking no
+    UserKnownHostsFile=/dev/null
+
+{% for host in groups['all'] | difference(groups['bastions'][0]) %}
+
+Host {{ host }}
+    Hostname {{ hostvars[host].ansible_host }}
+    ProxyCommand {{ ssh_proxy_command  }} -W {{ hostvars[host].private_v4 }}:22
+    IdentityFile {{ hostvars[host].ansible_private_key_file }}
+    User {{ ssh_user }}
+    StrictHostKeyChecking no
+    UserKnownHostsFile=/dev/null
+
+{% endfor %}