Create ansible role for deploying prometheus on openshift

A new role for installing prometheus on openshift.
Depends on `openshift_hosted_prometheus_deploy` flag

role creates:
- prometheus namespace
- prometheus clusterrolebinding and service account
- pvs for prometheus, alertmanager and alertbuffer for internal nfs
- prometheus pod with prometheus behind oauth-proxy, alertmanager and alert-buffer behind oauth-proxy
- prometheus and alertmanager configmaps
- prometheus and alerts services and direct routes
- prometheus, alertmanager and alert-buffer pvcs

4 files changed, 347 insertions, 0 deletions

diff --git a/roles/openshift_prometheus/tasks/create_pvs.yaml b/roles/openshift_prometheus/tasks/create_pvs.yaml
new file mode 100644
index 000000000..4e79da05f
--- /dev/null
+++ b/roles/openshift_prometheus/tasks/create_pvs.yaml
@@ -0,0 +1,36 @@
+---
+# Check for existance and then conditionally:
+# - evaluate templates
+# - PVs
+#
+# These tasks idempotently create required Prometheus PV objects. Do not
+# call this file directly. This file is intended to be ran as an
+# include that has a 'with_items' attached to it. Hence the use below
+# of variables like "{{ item.pv_label }}"
+
+- name: "Check if the {{ item.pv_label }} template has been created already"
+  oc_obj:
+    namespace: "{{ openshift_prometheus_namespace }}"
+    state: list
+    kind: pv
+    name: "{{ item.pv_name }}"
+  register: prom_pv_check
+
+# Skip all of this if the PV already exists
+- block:
+    - name: "Ensure the {{ item.pv_label }} template is evaluated"
+      template:
+        src: "{{ item.pv_template }}.j2"
+        dest: "{{ tempdir }}/templates/{{ item.pv_template }}"
+
+    - name: "Ensure {{ item.pv_label }} is created"
+      oc_obj:
+        namespace: "{{ openshift_prometheus_namespace }}"
+        kind: pv
+        name: "{{ item.pv_name }}"
+        state: present
+        delete_after: True
+        files:
+          - "{{ tempdir }}/templates/{{ item.pv_template }}"
+  when:
+    - not prom_pv_check.results.results.0
diff --git a/roles/openshift_prometheus/tasks/install_prometheus.yaml b/roles/openshift_prometheus/tasks/install_prometheus.yaml
new file mode 100644
index 000000000..93bdda3e8
--- /dev/null
+++ b/roles/openshift_prometheus/tasks/install_prometheus.yaml
@@ -0,0 +1,241 @@
+---
+
+# namespace
+- name: Add prometheus project
+  oc_project:
+    state: "{{ state }}"
+    name: "{{ openshift_prometheus_namespace }}"
+    node_selector: "{{ openshift_prometheus_node_selector | oo_selector_to_string_list() }}"
+    description: Prometheus
+
+# secrets
+- name: Set alert and prometheus secrets
+  oc_secret:
+    state: "{{ state }}"
+    name: "{{ item }}-proxy"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    contents:
+      - path: session_secret
+        data: "{{ 43 | oo_random_word }}="
+  with_items:
+    - prometheus
+    - alerts
+
+# serviceaccount
+- name: create prometheus serviceaccount
+  oc_serviceaccount:
+    state: "{{ state }}"
+    name: prometheus
+    namespace: "{{ openshift_prometheus_namespace }}"
+    #    TODO add annotations when supproted
+    #    annotations:
+    #      serviceaccounts.openshift.io/oauth-redirectreference.prom: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"prometheus"}}'
+    #      serviceaccounts.openshift.io/oauth-redirectreference.alerts: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alerts"}}'
+
+    secrets:
+      - prometheus-secrets
+  changed_when: no
+
+# TODO remove this when annotations are supported by oc_serviceaccount
+- name: annotate serviceaccount
+  command: >
+    {{ openshift.common.client_binary }} annotate --overwrite -n {{ openshift_prometheus_namespace }}
+    serviceaccount prometheus
+    serviceaccounts.openshift.io/oauth-redirectreference.prom='{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"prometheus"}}'
+    serviceaccounts.openshift.io/oauth-redirectreference.alerts='{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"alerts"}}'
+
+
+# create clusterrolebinding for prometheus serviceaccount
+- name: Set cluster-reader permissions for prometheus
+  oc_adm_policy_user:
+    state: "{{ state }}"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    resource_kind: cluster-role
+    resource_name: cluster-reader
+    user: "system:serviceaccount:{{ openshift_prometheus_namespace }}:prometheus"
+
+
+######################################################################
+# NFS
+# In the case that we are not running on a cloud provider, volumes must be statically provisioned
+
+- include: nfs.yaml
+  when: not (openshift_cloudprovider_kind is defined and (openshift_cloudprovider_kind == 'aws' or openshift_cloudprovider_kind == 'gce'))
+
+
+# create prometheus and alerts services
+# TODO join into 1 task with loop
+- name: Create prometheus service
+  oc_service:
+    state: "{{ state }}"
+    name: "{{ item.name }}"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    selector:
+      app: prometheus
+    labels:
+      name: "{{ item.name }}"
+      #    TODO add annotations when supported
+      #    annotations:
+      #      service.alpha.openshift.io/serving-cert-secret-name: "{{item.name}}-tls"
+    ports:
+      - port: 443
+        targetPort: 8443
+  with_items:
+    - name: prometheus
+
+- name: Create alerts service
+  oc_service:
+    state: "{{ state }}"
+    name: "{{ item.name }}"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    selector:
+      app: prometheus
+    labels:
+      name: "{{ item.name }}"
+      #    TODO add annotations when supported
+      #    annotations:
+      #      service.alpha.openshift.io/serving-cert-secret-name: "{{item.name}}-tls"
+    ports:
+      - port: 443
+        targetPort: 9443
+  with_items:
+    - name: alerts
+
+
+# Annotate services with secret name
+# TODO remove this when annotations are supported by oc_service
+- name: annotate prometheus service
+  command: >
+    {{ openshift.common.client_binary }} annotate --overwrite -n {{ openshift_prometheus_namespace }}
+    service prometheus 'service.alpha.openshift.io/serving-cert-secret-name=prometheus-tls'
+
+- name: annotate alerts service
+  command: >
+    {{ openshift.common.client_binary }} annotate --overwrite -n {{ openshift_prometheus_namespace }}
+    service alerts 'service.alpha.openshift.io/serving-cert-secret-name=prometheus-alerts-tls'
+
+# create prometheus and alerts routes
+- name: create prometheus and alerts routes
+  oc_route:
+    state: "{{ state }}"
+    name: "{{ item.name }}"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    service_name: "{{ item.name }}"
+    tls_termination: reencrypt
+  with_items:
+    - name: prometheus
+    - name: alerts
+
+# Storage
+- name: create prometheus pvc
+  oc_pvc:
+    namespace: "{{ openshift_prometheus_namespace }}"
+    name: "{{ openshift_prometheus_pvc_name }}"
+    access_modes: "{{ openshift_prometheus_pvc_access_modes }}"
+    volume_capacity: "{{ openshift_prometheus_pvc_size }}"
+    selector: "{{ openshift_prometheus_pvc_pv_selector }}"
+
+- name: create alertmanager pvc
+  oc_pvc:
+    namespace: "{{ openshift_prometheus_namespace }}"
+    name: "{{ openshift_prometheus_alertmanager_pvc_name }}"
+    access_modes: "{{ openshift_prometheus_alertmanager_pvc_access_modes }}"
+    volume_capacity: "{{ openshift_prometheus_alertmanager_pvc_size }}"
+    selector: "{{ openshift_prometheus_alertmanager_pvc_pv_selector }}"
+
+- name: create alertbuffer pvc
+  oc_pvc:
+    namespace: "{{ openshift_prometheus_namespace }}"
+    name: "{{ openshift_prometheus_alertbuffer_pvc_name }}"
+    access_modes: "{{ openshift_prometheus_alertbuffer_pvc_access_modes }}"
+    volume_capacity: "{{ openshift_prometheus_alertbuffer_pvc_size }}"
+    selector: "{{ openshift_prometheus_alertbuffer_pvc_pv_selector }}"
+
+# create prometheus deployment
+- name: Set prometheus deployment template
+  template:
+    src: prometheus_deployment.j2
+    dest: "{{ tempdir }}/templates/prometheus.yaml"
+  vars:
+    namespace: "{{ openshift_prometheus_namespace }}"
+    prom_replicas: "{{ openshift_prometheus_replicas }}"
+
+- name: Set prometheus deployment
+  oc_obj:
+    state: "{{ state }}"
+    name: "prometheus"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    kind: deployment
+    files:
+      - "{{ tempdir }}/templates/prometheus.yaml"
+    delete_after: true
+
+# prometheus configmap
+# Copy the additional rules file if it is defined
+- name: Copy additional rules file to host
+  copy:
+    src: "{{ openshift_prometheus_additional_rules_file }}"
+    dest: "{{ tempdir }}/prometheus.additional.rules"
+  when:
+    - openshift_prometheus_additional_rules_file is defined
+    - openshift_prometheus_additional_rules_file is not none
+    - openshift_prometheus_additional_rules_file | trim | length > 0
+
+- stat:
+    path: "{{ tempdir }}/prometheus.additional.rules"
+  register: additional_rules_stat
+
+# The kubernetes version impacts the prometheus scraping endpoint
+# so gathering it before constructing the configmap
+- name: get oc version
+  oc_version:
+  register: oc_version
+
+- set_fact:
+    kubernetes_version: "{{ oc_version.results.kubernetes_short | float }}"
+
+- template:
+    src: prometheus.yml.j2
+    dest: "{{ tempdir }}/prometheus.yml"
+  changed_when: no
+
+- template:
+    src: prometheus.rules.j2
+    dest: "{{ tempdir }}/prometheus.rules"
+  changed_when: no
+
+# In prometheus configmap create "additional.rules" section if file exists
+- name: Set prometheus configmap
+  oc_configmap:
+    state: "{{ state }}"
+    name: "prometheus"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    from_file:
+      prometheus.rules: "{{ tempdir }}/prometheus.rules"
+      prometheus.additional.rules: "{{ tempdir }}/prometheus.additional.rules"
+      prometheus.yml: "{{ tempdir }}/prometheus.yml"
+  when: additional_rules_stat.stat.exists == True
+
+- name: Set prometheus configmap
+  oc_configmap:
+    state: "{{ state }}"
+    name: "prometheus"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    from_file:
+      prometheus.rules: "{{ tempdir }}/prometheus.rules"
+      prometheus.yml: "{{ tempdir }}/prometheus.yml"
+  when: additional_rules_stat.stat.exists == False
+
+# alertmanager configmap
+- template:
+    src: alertmanager.yml.j2
+    dest: "{{ tempdir }}/alertmanager.yml"
+  changed_when: no
+
+- name: Set alertmanager configmap
+  oc_configmap:
+    state: "{{ state }}"
+    name: "prometheus-alerts"
+    namespace: "{{ openshift_prometheus_namespace }}"
+    from_file:
+      alertmanager.yml: "{{ tempdir }}/alertmanager.yml"
diff --git a/roles/openshift_prometheus/tasks/main.yaml b/roles/openshift_prometheus/tasks/main.yaml
new file mode 100644
index 000000000..523a64334
--- /dev/null
+++ b/roles/openshift_prometheus/tasks/main.yaml
@@ -0,0 +1,26 @@
+---
+
+- name: Create temp directory for doing work in on target
+  command: mktemp -td openshift-prometheus-ansible-XXXXXX
+  register: mktemp
+  changed_when: False
+
+- set_fact:
+    tempdir: "{{ mktemp.stdout }}"
+
+- name: Create templates subdirectory
+  file:
+    state: directory
+    path: "{{ tempdir }}/templates"
+    mode: 0755
+  changed_when: False
+
+- include: install_prometheus.yaml
+  vars:
+    state: "{{ openshift_prometheus_state }}"
+
+- name: Delete temp directory
+  file:
+    name: "{{ tempdir }}"
+    state: absent
+  changed_when: False
diff --git a/roles/openshift_prometheus/tasks/nfs.yaml b/roles/openshift_prometheus/tasks/nfs.yaml
new file mode 100644
index 000000000..0b45f2cee
--- /dev/null
+++ b/roles/openshift_prometheus/tasks/nfs.yaml
@@ -0,0 +1,44 @@
+---
+# Tasks to statically provision NFS volumes
+# Include if not using dynamic volume provisioning
+- name: Ensure the /exports/ directory exists
+  file:
+    path: /exports/
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+
+- name: Ensure the prom-pv0X export directories exist
+  file:
+    path: "/exports/{{ item }}"
+    state: directory
+    mode: 0777
+    owner: nfsnobody
+    group: nfsnobody
+  with_items: "{{ openshift_prometheus_pv_exports }}"
+
+- name: Ensure the NFS exports for Prometheus PVs exist
+  copy:
+    src: openshift_prometheus.exports
+    dest: /etc/exports.d/openshift_prometheus.exports
+  register: nfs_exports_updated
+
+- name: Ensure the NFS export table is refreshed if exports were added
+  command: exportfs -ar
+  when:
+    - nfs_exports_updated.changed
+
+
+######################################################################
+# Create the required Prometheus PVs. Check out these online docs if you
+# need a refresher on includes looping with items:
+# * http://docs.ansible.com/ansible/playbooks_loops.html#loops-and-includes-in-2-0
+# * http://stackoverflow.com/a/35128533
+#
+# TODO: Handle the case where a PV template is updated in
+# openshift-ansible and the change needs to be landed on the managed
+# cluster.
+
+- include: create_pvs.yaml
+  with_items: "{{ openshift_prometheus_pv_data }}"