diff options
32 files changed, 322 insertions, 47 deletions
@@ -1 +1,2 @@ *.retry +.vault-pass diff --git a/common.yml b/common.yml new file mode 100644 index 0000000..6fc2c9f --- /dev/null +++ b/common.yml @@ -0,0 +1,5 @@ +- name: Common Software + hosts: all + remote_user: root + roles: + - role: common diff --git a/group_vars/all.yml b/group_vars/all.yml new file mode 100644 index 0000000..5b15a48 --- /dev/null +++ b/group_vars/all.yml @@ -0,0 +1,16 @@ +ands_domain: ands.kit.edu +public_domain: ipe.kit.edu +public_search_domains: "{{ public_domain }},kaas.kit.edu" +ands_network: 192.168.26.0/24 +public_network: 141.52.64.0/23 + +ands_compute_baseip: 130 +ands_storage_baseip: 140 + +ands_idm_server_ip: "{{ ands_network | ipaddr(212) | ipaddr('address') }}" +ands_pdv_server_ip: "{{ ands_network | ipaddr(170) | ipaddr('address') }}" +ands_nfs_server_ip: "{{ ands_network | ipaddr(140) | ipaddr('address') }}" + +ands_idm_server: "ipeidm.{{ ands_domain }}" +ands_pdv_server: "{{ ands_pdv_server_ip }}" +ands_nfs_server: "{{ ands_nfs_server_ip }}" @@ -0,0 +1,5 @@ +- name: IDM Infrastructure + hosts: ipaclients + remote_user: root + roles: + - role: ands_idm diff --git a/install.yml b/install.yml index f1acdd8..802dc10 100644 --- a/install.yml +++ b/install.yml @@ -4,6 +4,12 @@ roles: - role: common +- name: IDM infrastructure + hosts: ipaclients + remote_user: root + roles: + - role: ands_idm + - name: Infiniband hosts: ib remote_user: root @@ -23,7 +29,6 @@ # roles: # - role: rocm - - name: Docker hosts: docker remote_user: root @@ -40,4 +45,4 @@ hosts: infra remote_user: root roles: - - role: storage + - role: ands_storage diff --git a/inventories/ipe.erb b/inventories/ipe.erb index 20edf72..2ff9e24 100644 --- a/inventories/ipe.erb +++ b/inventories/ipe.erb @@ -34,3 +34,10 @@ student 192.168.26.[80:89] 192.168.26.[130:139] 192.168.26.[140:149] + +[vm] +192.168.26.216 + +[ipaclients:children] +vm +compute diff --git a/roles/ands_idm/defaults/main.yml b/roles/ands_idm/defaults/main.yml new file mode 100644 index 0000000..07e67af --- /dev/null +++ b/roles/ands_idm/defaults/main.yml @@ -0,0 +1 @@ +ands_none: "{{ None }}" diff --git a/roles/ands_idm/tasks/find_ands_connection.yml b/roles/ands_idm/tasks/find_ands_connection.yml new file mode 100644 index 0000000..f4cf9b6 --- /dev/null +++ b/roles/ands_idm/tasks/find_ands_connection.yml @@ -0,0 +1,18 @@ +- name: "Detect ands network interface" + include_tasks: "find_interface_by_net.yml" + vars: + var: "ands_network_interface" + net: "{{ ands_network }}" + when: + - ands_network_interface is not defined + - ands_network is defined + + +- name: "Detect ands network connection" + include_tasks: "find_connection_by_if.yml" + vars: + var: "ands_network_connection" + iface: "{{ ands_network_interface }}" + when: + - ands_network_connection is not defined + - ands_network_interface is defined diff --git a/roles/ands_idm/tasks/find_connection_by_if.yml b/roles/ands_idm/tasks/find_connection_by_if.yml new file mode 100644 index 0000000..3fd883e --- /dev/null +++ b/roles/ands_idm/tasks/find_connection_by_if.yml @@ -0,0 +1,9 @@ +- name: "Detect nm connection corresponding to interface '{{ bridge | default(iface) }}'" + shell: "nmcli d show {{ iface | quote }} | grep CONNECTION | cut -d ':' -f 2- | sed -E -e 's/^[[:space:]]+//' | grep '^[[:alpha:]]'" + register: conres + failed_when: false + changed_when: false + +- name: "Set {{ var }} to {{ conres.stdout | quote }}" + set_fact: + "{{ var }}": "{{ conres.stdout }}" diff --git a/roles/ands_idm/tasks/find_interface_by_net.yml b/roles/ands_idm/tasks/find_interface_by_net.yml new file mode 100644 index 0000000..ad44578 --- /dev/null +++ b/roles/ands_idm/tasks/find_interface_by_net.yml @@ -0,0 +1,17 @@ +- name: "Looking for interface holding {{ net }}" + set_fact: + "{{ var }}": "{{ eth['device'] }}" + vars: + eth: "{{ hostvars[inventory_hostname]['ansible_' + item] | default({}) }}" + ipv4: "{{ eth['ipv4'] | default({}) }}" + q: "{{ eth | json_query('ipv4_secondaries[*].network') }}" + sec: "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}" + nets: "{{ sec | union([ipv4.network]) }}" + when: + - eth['type'] is defined + - eth['ipv4'] is defined + - eth['device'] is defined + - eth['type'] == 'ether' + - net | ipaddr('network') in nets + with_items: + - "{{ hostvars[inventory_hostname]['ansible_interfaces'] }}" diff --git a/roles/ands_idm/tasks/main.yml b/roles/ands_idm/tasks/main.yml new file mode 100644 index 0000000..667f14e --- /dev/null +++ b/roles/ands_idm/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- include_vars: dir="config" name="config" + +- name: Setup DNS + include_tasks: "setup_dns.yml" + +- name: Setup ipa-client + include_tasks: "setup_ipa.yml" + diff --git a/roles/ands_idm/tasks/setup_dns.yml b/roles/ands_idm/tasks/setup_dns.yml new file mode 100644 index 0000000..a463c77 --- /dev/null +++ b/roles/ands_idm/tasks/setup_dns.yml @@ -0,0 +1,38 @@ +- name: "Find NM connection" + include_tasks: "find_ands_connection.yml" + +- name: "Change FQDN" + replace: path="/etc/hostname" regexp="{{ public_domain }}" replace="{{ ands_domain }}" + +- name: "Read FQDN" + command: "cat /etc/hostname" + register: hostname + changed_when: false + +- name: "Adjust runtime FQDN" + hostname: name="{{ hostname.stdout }}" + +- name: "Find configured DNS servers" + shell: "nmcli d show {{ ands_network_interface }} | grep DNS | grep {{ ands_idm_server_ip | quote }}" + register: dns_check + changed_when: dns_check is failed + failed_when: false + +- name: "Change DNS server on {{ ands_network_connection }}" +# nmcli: conn_name="{{ ands_network_connection }}" dns4="[{{ ands_idm_server_ip }}]" state="present" + command: "nmcli connection modify {{ ands_network_connection | quote }} ipv4.dns {{ ands_idm_server_ip }} ipv4.ignore-auto-dns yes ipv6.ignore-auto-dns yes ipv4.dns-search '{{ ands_domain,public_search_domains }}'" + register: result + when: + - ands_network_connection is defined + - dns_check.rc != 0 + +- name: "Update associated interface {{ ands_network_interface }}" + command: "nmcli connection up {{ ands_network_connection | quote }}" + when: + - ands_network_interface is defined + - result is changed + + + +#- name: Register idM in /etc/hosts +# lineinfile: dest="/etc/hosts" line="192.168.26.212 ipeidm.ands.kit.edu ipeidm" regexp="ipeidm$" state="present" diff --git a/roles/ands_idm/tasks/setup_ipa.yml b/roles/ands_idm/tasks/setup_ipa.yml new file mode 100644 index 0000000..06fe6da --- /dev/null +++ b/roles/ands_idm/tasks/setup_ipa.yml @@ -0,0 +1,20 @@ +- name: Install ipa-client + package: name=ipa-client state=present + +- name: "Check if ipa is already configured" + shell: "grep {{ ands_domain }} /etc/krb5.conf" + register: ipa_check + changed_when: ipa_check is failed + failed_when: false + +- name: "Read FQDN" + command: "cat /etc/hostname" + register: hostname + changed_when: false + +- name: "Configure ipa" + command: "/usr/sbin/ipa-client-install -p admin -w {{ config.ands_idm_admin_password | quote }} --hostname={{ hostname.stdout }} --domain={{ ands_domain }} --server={{ ands_idm_server }} --enable-dns-updates --mkhomedir --request-cert --unattended" + when: + - ipa_check.rc != 0 + + diff --git a/roles/ands_idm/vars/config b/roles/ands_idm/vars/config new file mode 120000 index 0000000..a2a1973 --- /dev/null +++ b/roles/ands_idm/vars/config @@ -0,0 +1 @@ +../../../setup/config/
\ No newline at end of file diff --git a/roles/ands_kitauth/README b/roles/ands_kitauth/README new file mode 100644 index 0000000..d2e820b --- /dev/null +++ b/roles/ands_kitauth/README @@ -0,0 +1,20 @@ +Tasks +===== + - required packages: ssd-ldap + * nice tool to manage stuff is realmd (but it only can be used by Activer directory admins, so not for KIT) + + - prepare space for home directories + * /home/kit.edu should be created (and optionally mounted to NFS) + + - Automate home creation + * Either run + authconfig --enablemkhomedir --update + * Or copy script and add in the end of /etc/pam/sshd + session optional pam_exec.so /usr/local/bin/login_script.sh + + +Considerations +============== + - sssd files should belong to root and has 0600 access. + + diff --git a/roles/ands_kitauth/files/scripts/login_script.sh b/roles/ands_kitauth/files/scripts/login_script.sh new file mode 100755 index 0000000..9b66968 --- /dev/null +++ b/roles/ands_kitauth/files/scripts/login_script.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Script to authomatecly create user home directories +# Shall we consider creating both NFS and local homes? Or shall we just create it on NFS? + +LOG=/var/log/login_script +DATE=`/bin/date +"%b %d %H:%M:%S"` + +if [ x${PAM_TYPE} == "xopen_session" ]; then set $(getent passwd $PAM_USER | awk -F ":" '{print $3" "$4" "$5" "$6}') + USERUID=$1; USERGID=$2; USERHOME=$4 + + echo "[$DATE]: login of $PAM_USER: UID=$USERUID: GID=$USERGID: HOME=$USERHOME: from $PAM_RHOST via $PAM_SERVICE" >> $LOG + + if [ ! -z "${USERHOME}" -a ! -d ${USERHOME} ]; then + #if [ ! -d /home/${USERHOME} ]; then + echo " Home for user ${USERNAME} does not exist at ${USERHOME} => creating" >> $LOG + + mkdir -p $USERHOME + cp -af /etc/skel/.[a-zA-Z]* $USERHOME + chown -R $USERUID.$USERGID $USERHOME + chmod 701 $USERHOME + chmod g+s $USERHOME + fi +fi +if [ x${PAM_TYPE} == "xclose_session" ]; then + echo "[$DATE]: logout of $PAM_USER $USERUID" >> $LOG +fi diff --git a/roles/ands_kitauth/files/sssd/kit.conf b/roles/ands_kitauth/files/sssd/kit.conf new file mode 100644 index 0000000..f4aee29 --- /dev/null +++ b/roles/ands_kitauth/files/sssd/kit.conf @@ -0,0 +1,17 @@ +[domain/kit.edu] +ldap_tls_reqcert = allow +ldap_id_use_start_tls = true +cache_credentials = true +auth_provider = ldap +chpass_provider = ldap +id_provider = ldap +ldap_uri = ldap://bwidm.scc.kit.edu:389/ +ldap_search_base = ou=lsdf-dis,dc=bwlsdf,dc=de +ldap_default_bind_dn = uid=fileservice-read,ou=admin,ou=lsdf-dis,dc=bwlsdf,dc=de +ldap_default_authtok_type = password +ldap_default_authtok = H7fjmJhvr58hjbv411fmjuhb +ldap_tls_cacertdir = /etc/openldap/certs +#ldap_user_home_directory = homeDirectory +override_homedir = /home/%d/%u +debug_level = 10 +debug_timestamps = true diff --git a/roles/ands_kitauth/files/sssd/sssd.conf b/roles/ands_kitauth/files/sssd/sssd.conf new file mode 100644 index 0000000..af34dba --- /dev/null +++ b/roles/ands_kitauth/files/sssd/sssd.conf @@ -0,0 +1,15 @@ +[sssd] +config_file_version = 2 +services = nss, pam +domains = kit.edu + +[nss] +filter_groups = root +filter_users = root +entry_cache_timeout = 300 +entry_cache_nowait_percentage = 75 + +[pam] +offline_credentials_expiration = 2 +offline_failed_login_attempts = 3 +offline_failed_login_delay = 5 diff --git a/roles/ands_storage/defaults/main.yml b/roles/ands_storage/defaults/main.yml new file mode 100644 index 0000000..af631f8 --- /dev/null +++ b/roles/ands_storage/defaults/main.yml @@ -0,0 +1,5 @@ +compute2_ip: "{{ ands_network | ipaddr(ands_compute_baseip + 2) | ipaddr('address') }}" +compute4_ip: "{{ ands_network | ipaddr(ands_compute_baseip + 4) | ipaddr('address') }}" + +compute4_ssds: ['sda','sdb','sdc','sdd','sde','sdf','sdg','sdh'] + diff --git a/roles/storage/tasks/ipecompute2.yml b/roles/ands_storage/tasks/ipecompute2.yml index 9b2cef8..9b2cef8 100644 --- a/roles/storage/tasks/ipecompute2.yml +++ b/roles/ands_storage/tasks/ipecompute2.yml diff --git a/roles/storage/tasks/ipecompute4.yml b/roles/ands_storage/tasks/ipecompute4.yml index 5b3a88f..c226519 100644 --- a/roles/storage/tasks/ipecompute4.yml +++ b/roles/ands_storage/tasks/ipecompute4.yml @@ -5,6 +5,11 @@ # - "{{ compute4_ssds }}" # - [ 2, 3, 4 ] +- name: Find SSD devices + shell: "ls /dev/disk/by-id/*SSD_850_PRO* | grep -v part" + changed_when: false + register: ssds + - name: Create partition parted: device: "/dev/{{ item }}" @@ -14,7 +19,8 @@ flags: [raid] state: "present" failed_when: false - with_items: "{{ compute4_ssds }}" + with_items: "{{ ssds.stdout_lines }}" +# with_items: "{{ compute4_ssds }}" - name: arrays | Checking Status Of Array(s) shell: "cat /proc/mdstat | grep md10" @@ -24,7 +30,8 @@ check_mode: no - name: arrays | Creating Array(s) - shell: "yes | mdadm --create /dev/md10 --level=0 --raid-devices={{ compute4_ssds | count }} {{ compute4_ssds | map('regex_replace', '(.*)', '/dev/\\1') | join ('1 ') }}1" +# shell: "yes | mdadm --create /dev/md10 --level=0 --raid-devices={{ compute4_ssds | count }} {{ compute4_ssds | map('regex_replace', '(.*)', '/dev/\\1') | join ('1 ') }}1" + shell: "yes | mdadm --create /dev/md10 --level=0 --raid-devices={{ ssds.stdout_lines | count }} {{ ssds.stdout_lines | join ('1 ') }}1" register: "array_created" when: array_check.rc != 0 diff --git a/roles/storage/tasks/main.yml b/roles/ands_storage/tasks/main.yml index 014e396..351b529 100644 --- a/roles/storage/tasks/main.yml +++ b/roles/ands_storage/tasks/main.yml @@ -3,15 +3,13 @@ package: name="{{ item }}" state=present with_items: [ 'parted', 'mdadm', 'nfs-utils' ] -- debug: msg="{{ inventory_hostname }}" - - name: configure network fs include_tasks: nfs.yml - name: configure ipepdvcompute2 include_tasks: ipecompute2.yml - when: inventory_hostname == '192.168.26.132' + when: inventory_hostname == compute2_ip - name: configure ipepdvcompute4 include_tasks: ipecompute4.yml - when: inventory_hostname == '192.168.26.134' + when: inventory_hostname == compute4_ip diff --git a/roles/ands_storage/tasks/nfs.yml b/roles/ands_storage/tasks/nfs.yml new file mode 100644 index 0000000..f8bb310 --- /dev/null +++ b/roles/ands_storage/tasks/nfs.yml @@ -0,0 +1,15 @@ +--- +- name: Create mountable dir + file: path=/mnt/ands state=directory mode=755 owner=root group=root + +- name: Create mountable dir + file: path=/mnt/pdv state=directory mode=755 owner=root group=root + +- name: set mountpoints + mount: name=/mnt/ands src="{{ ands_nfs_server }}:/mnt/ands" fstype=nfs4 opts=defaults,minorversion=1,_netdev,nofail,soft,nodiratime,noatime dump=0 passno=0 state="{{ ands_mount_nfs | default(false) | ternary ('mounted', 'absent') }}" + when: ands_nfs_server is defined + +- name: set mountpoints + mount: name=/mnt/pdv src="{{ ands_pdv_server }}:/pdv" fstype=nfs opts=defaults,_netdev,nofail,soft,nodiratime,noatime dump=0 passno=0 state=mounted + when: ands_pdv_server is defined + diff --git a/roles/common/tasks/install.yml b/roles/common/tasks/install.yml new file mode 100644 index 0000000..9f3cf79 --- /dev/null +++ b/roles/common/tasks/install.yml @@ -0,0 +1,24 @@ +- name: Ensure all required repositories are configured + package: name={{item}} state=present + register: result + with_items: + - epel-release + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + +- include_tasks: main_yum.yml + when: ansible_pkg_mgr == 'yum' + +- include_tasks: main_dnf.yml + when: ansible_pkg_mgr == 'dnf' + +- name: Install additional software + include_tasks: software.yml + +- name: Configure git + shell: | + git config --global http.sslVerify false + git config --global user.name "Suren A. Chilingaryan" + git config --global user.email csa@suren.me + exit 0 + args: + executable: /bin/bash diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 9f3cf79..9f0b72c 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -1,24 +1,3 @@ -- name: Ensure all required repositories are configured - package: name={{item}} state=present - register: result - with_items: - - epel-release - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' - -- include_tasks: main_yum.yml - when: ansible_pkg_mgr == 'yum' - -- include_tasks: main_dnf.yml - when: ansible_pkg_mgr == 'dnf' - -- name: Install additional software - include_tasks: software.yml - -- name: Configure git - shell: | - git config --global http.sslVerify false - git config --global user.name "Suren A. Chilingaryan" - git config --global user.email csa@suren.me - exit 0 - args: - executable: /bin/bash +--- +- name: "Configuring OpenShift" + include_tasks: "{{ subrole | default('install') }}.yml" diff --git a/roles/common/tasks/main_dnf.yml b/roles/common/tasks/main_dnf.yml index 0572132..792a52a 100644 --- a/roles/common/tasks/main_dnf.yml +++ b/roles/common/tasks/main_dnf.yml @@ -5,7 +5,7 @@ # We always update on first install and if requested - name: Update CentOS dnf: name=* state=latest - when: (result | changed) or (os_update | default(false)) + when: (result is changed) or (os_update | default(false)) - name: Install various ansible requirements package: name={{item}} state=present diff --git a/roles/common/tasks/update.yml b/roles/common/tasks/update.yml new file mode 100644 index 0000000..db8ae39 --- /dev/null +++ b/roles/common/tasks/update.yml @@ -0,0 +1,16 @@ +- name: Ensure all required repositories are configured + package: name={{item}} state=present + register: result + with_items: + - epel-release + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + +- include_tasks: main_yum.yml + when: ansible_pkg_mgr == 'yum' + vars: + os_update: true + +- include_tasks: main_dnf.yml + when: ansible_pkg_mgr == 'dnf' + vars: + os_update: true diff --git a/roles/storage/defaults/main.yml b/roles/storage/defaults/main.yml deleted file mode 100644 index ca36e70..0000000 --- a/roles/storage/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ -compute4_ssds: ['sda','sdb','sdc','sdd','sde','sdf','sdg','sdh'] diff --git a/roles/storage/tasks/nfs.yml b/roles/storage/tasks/nfs.yml deleted file mode 100644 index 9dbd467..0000000 --- a/roles/storage/tasks/nfs.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Create mountable dir - file: path=/mnt/ands state=directory mode=755 owner=root group=root - -- name: Create mountable dir - file: path=/mnt/pdv state=directory mode=755 owner=root group=root - -- name: set mountpoints - mount: name=/mnt/ands src=192.168.26.140:/mnt/ands fstype=nfs4 opts=defaults,minorversion=1,_netdev,nofail,soft,nodiratime,noatime dump=0 passno=0 state=absent - -- name: set mountpoints - mount: name=/mnt/pdv src=192.168.26.170:/pdv fstype=nfs opts=defaults,_netdev,nofail,soft,nodiratime,noatime dump=0 passno=0 state=mounted diff --git a/setup/config/secret.yml b/setup/config/secret.yml new file mode 100644 index 0000000..9db33f3 --- /dev/null +++ b/setup/config/secret.yml @@ -0,0 +1,7 @@ +$ANSIBLE_VAULT;1.1;AES256 +66303864363537356532616538616262303466383234366138333938613763303532313832336133 +3330386338383635343833343238663833613035303030620a383663643135633931373436363731 +39636230363534363237383334376432366338313837623461623733663566653631646135326337 +3962626333633763300a383831666234653466373435613337313834316534396163346632356236 +64613161323762383638393830323864376239623831643866646532306238333833346133356561 +3936633465643336633262346230333038316435333635313035 diff --git a/storage.yml b/storage.yml index 787f0b5..318ecaf 100644 --- a/storage.yml +++ b/storage.yml @@ -2,5 +2,5 @@ hosts: all remote_user: root roles: - - role: storage + - role: ands_storage diff --git a/update.yml b/update.yml new file mode 100644 index 0000000..d719914 --- /dev/null +++ b/update.yml @@ -0,0 +1,7 @@ +- name: Common Software + hosts: all + remote_user: root + roles: + - role: common + vars: + subrole: update |