Configure Kuryr CNI daemon

Kuryr CNI daemon is scalability improvement that moves watching K8s API
and VIF plugging into a separate entity called kuryr-daemon.
Kuryr-daemon will run in a container and serve requests from
kuryr-driver.

3 files changed, 355 insertions, 23 deletions

diff --git a/roles/kuryr/tasks/node.yaml b/roles/kuryr/tasks/node.yaml
index 08f2d5adc..41d0ead20 100644
--- a/roles/kuryr/tasks/node.yaml
+++ b/roles/kuryr/tasks/node.yaml
@@ -40,7 +40,7 @@
     regexp: '^OPTIONS="?(.*?)"?$'
     backrefs: yes
     backup: yes
-    line: 'OPTIONS="\1 --disable dns,proxy,plugins"'
+    line: 'OPTIONS="\1 --disable proxy"'
 
 - name: force node restart to disable the proxy
   service:
diff --git a/roles/kuryr/templates/cni-daemonset.yaml.j2 b/roles/kuryr/templates/cni-daemonset.yaml.j2
index 39348ae90..09f4c7dfe 100644
--- a/roles/kuryr/templates/cni-daemonset.yaml.j2
+++ b/roles/kuryr/templates/cni-daemonset.yaml.j2
@@ -26,6 +26,13 @@ spec:
         image: kuryr/cni:latest
         imagePullPolicy: IfNotPresent
         command: [ "cni_ds_init" ]
+        env:
+        - name: CNI_DAEMON
+          value: "True"
+        - name: KUBERNETES_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         securityContext:
           privileged: true
         volumeMounts:
@@ -38,6 +45,10 @@ spec:
           subPath: kuryr-cni.conf
         - name: etc
           mountPath: /etc
+        - name: proc
+          mountPath: /host_proc
+        - name: openvswitch
+          mountPath: /var/run/openvswitch
       volumes:
         - name: bin
           hostPath:
@@ -50,4 +61,10 @@ spec:
             name: kuryr-config
         - name: etc
           hostPath:
-            path: /etc
\ No newline at end of file
+            path: /etc
+        - name: proc
+          hostPath:
+            path: /proc
+        - name: openvswitch
+          hostPath:
+            path: /var/run/openvswitch
diff --git a/roles/kuryr/templates/configmap.yaml.j2 b/roles/kuryr/templates/configmap.yaml.j2
index 96c215f00..4bf1dbddf 100644
--- a/roles/kuryr/templates/configmap.yaml.j2
+++ b/roles/kuryr/templates/configmap.yaml.j2
@@ -16,17 +16,17 @@ data:
     # Directory for Kuryr vif binding executables. (string value)
     #bindir = /usr/libexec/kuryr
 
+    # Neutron subnetpool name will be prefixed by this. (string value)
+    #subnetpool_name_prefix = kuryrPool
+
+    # baremetal or nested-containers are the supported values. (string value)
+    #deployment_type = baremetal
+
     # If set to true, the logging level will be set to DEBUG instead of the default
     # INFO level. (boolean value)
     # Note: This option can be changed without restarting.
     #debug = false
 
-    # DEPRECATED: If set to false, the logging level will be set to WARNING instead
-    # of the default INFO level. (boolean value)
-    # This option is deprecated for removal.
-    # Its value may be silently ignored in the future.
-    #verbose = true
-
     # The name of a logging configuration file. This file is appended to any
     # existing logging configuration files. For details about logging configuration
     # files, see the Python logging module documentation. Note that when logging
@@ -46,7 +46,7 @@ data:
     # logging will go to stderr as defined by use_stderr. This option is ignored if
     # log_config_append is set. (string value)
     # Deprecated group/name - [DEFAULT]/logfile
-    #log_file = /var/log/kuryr/kuryr-controller.log
+    #log_file = <None>
 
     # (Optional) The base directory used for relative log_file  paths. This option
     # is ignored if log_config_append is set. (string value)
@@ -65,13 +65,19 @@ data:
     # is set. (boolean value)
     #use_syslog = false
 
+    # Enable journald for logging. If running in a systemd environment you may wish
+    # to enable journal support. Doing so will use the journal native protocol
+    # which includes structured metadata in addition to log messages.This option is
+    # ignored if log_config_append is set. (boolean value)
+    #use_journal = false
+
     # Syslog facility to receive log lines. This option is ignored if
     # log_config_append is set. (string value)
     #syslog_log_facility = LOG_USER
 
     # Log output to standard error. This option is ignored if log_config_append is
     # set. (boolean value)
-    #use_stderr = true
+    #use_stderr = false
 
     # Format string to use for log messages with context. (string value)
     #logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
@@ -93,7 +99,7 @@ data:
 
     # List of package logging levels in logger=LEVEL pairs. This option is ignored
     # if log_config_append is set. (list value)
-    #default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO
+    #default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO
 
     # Enables or disables publication of error events. (boolean value)
     #publish_errors = false
@@ -106,15 +112,86 @@ data:
     # value)
     #instance_uuid_format = "[instance: %(uuid)s] "
 
+    # Interval, number of seconds, of log rate limiting. (integer value)
+    #rate_limit_interval = 0
+
+    # Maximum number of logged messages per rate_limit_interval. (integer value)
+    #rate_limit_burst = 0
+
+    # Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG
+    # or empty string. Logs with level greater or equal to rate_limit_except_level
+    # are not filtered. An empty string means that all levels are filtered. (string
+    # value)
+    #rate_limit_except_level = CRITICAL
+
     # Enables or disables fatal status of deprecations. (boolean value)
     #fatal_deprecations = false
 
 
     [binding]
+    # Configuration options for container interface binding.
 
-    driver = kuryr.lib.binding.drivers.vlan
+    #
+    # From kuryr_kubernetes
+    #
+
+    # The name prefix of the veth endpoint put inside the container. (string value)
+    #veth_dst_prefix = eth
+
+    # Driver to use for binding and unbinding ports. (string value)
+    # Deprecated group/name - [binding]/driver
+    #default_driver = kuryr.lib.binding.drivers.veth
+
+    # Drivers to use for binding and unbinding ports. (list value)
+    #enabled_drivers = kuryr.lib.binding.drivers.veth
+
+    # Specifies the name of the Nova instance interface to link the virtual devices
+    # to (only applicable to some binding drivers. (string value)
     link_iface = eth0
 
+    driver = kuryr.lib.binding.drivers.vlan
+
+
+    [cni_daemon]
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # Enable CNI Daemon configuration. (boolean value)
+    daemon_enabled = true
+
+    # Bind address for CNI daemon HTTP server. It is recommened to allow only local
+    # connections. (string value)
+    bind_address = 127.0.0.1:50036
+
+    # Maximum number of processes that will be spawned to process requests from CNI
+    # driver. (integer value)
+    #worker_num = 30
+
+    # Time (in seconds) the CNI daemon will wait for VIF annotation to appear in
+    # pod metadata before failing the CNI request. (integer value)
+    #vif_annotation_timeout = 120
+
+    # Kuryr uses pyroute2 library to manipulate networking interfaces. When
+    # processing a high number of Kuryr requests in parallel, it may take kernel
+    # more time to process all networking stack changes. This option allows to tune
+    # internal pyroute2 timeout. (integer value)
+    #pyroute2_timeout = 30
+
+    # Set to True when you are running kuryr-daemon inside a Docker container on
+    # Kubernetes host. E.g. as DaemonSet on Kubernetes cluster Kuryr is supposed to
+    # provide networking for. This mainly means thatkuryr-daemon will look for
+    # network namespaces in $netns_proc_dir instead of /proc. (boolean value)
+    docker_mode = true
+
+    # When docker_mode is set to True, this config option should be set to where
+    # host's /proc directory is mounted. Please note that mounting it is necessary
+    # to allow Kuryr-Kubernetes to move host interfaces between host network
+    # namespaces, which is essential for Kuryr to work. (string value)
+    netns_proc_dir = /host_proc
+
+
     [kubernetes]
 
     #
@@ -164,11 +241,6 @@ data:
     # The driver that manages VIFs pools for Kubernetes Pods (string value)
     vif_pool_driver = {{ kuryr_openstack_enable_pools | default(False) | ternary('nested', 'noop') }}
 
-    [vif_pool]
-    ports_pool_max = {{ kuryr_openstack_pool_max | default(0) }}
-    ports_pool_min = {{ kuryr_openstack_pool_min | default(1) }}
-    ports_pool_batch = {{ kuryr_openstack_pool_batch | default(5) }}
-    ports_pool_update_frequency = {{ kuryr_openstack_pool_update_frequency | default(20) }}
 
     [neutron]
     # Configuration options for OpenStack Neutron
@@ -232,13 +304,55 @@ data:
     external_svc_subnet = {{ kuryr_openstack_external_svc_subnet_id }}
 
     [pod_vif_nested]
+
     worker_nodes_subnet = {{ kuryr_openstack_worker_nodes_subnet_id }}
+
+
+    [pool_manager]
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # Absolute path to socket file that will be used for communication with the
+    # Pool Manager daemon (string value)
+    #sock_file = /run/kuryr/kuryr_manage.sock
+
+
+    [vif_pool]
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # Set a maximun amount of ports per pool. 0 to disable (integer value)
+    ports_pool_max = {{ kuryr_openstack_pool_max | default(0) }}
+
+    # Set a target minimum size of the pool of ports (integer value)
+    ports_pool_min = {{ kuryr_openstack_pool_min | default(1) }}
+
+    # Number of ports to be created in a bulk request (integer value)
+    ports_pool_batch = {{ kuryr_openstack_pool_batch | default(5) }}
+
+    # Minimun interval (in seconds) between pool updates (integer value)
+    ports_pool_update_frequency = {{ kuryr_openstack_pool_update_frequency | default(20) }}
+
   kuryr-cni.conf: |+
     [DEFAULT]
 
     #
     # From kuryr_kubernetes
     #
+
+    # Directory for Kuryr vif binding executables. (string value)
+    #bindir = /usr/libexec/kuryr
+
+    # Neutron subnetpool name will be prefixed by this. (string value)
+    #subnetpool_name_prefix = kuryrPool
+
+    # baremetal or nested-containers are the supported values. (string value)
+    #deployment_type = baremetal
+
     # If set to true, the logging level will be set to DEBUG instead of the default
     # INFO level. (boolean value)
     # Note: This option can be changed without restarting.
@@ -263,7 +377,7 @@ data:
     # logging will go to stderr as defined by use_stderr. This option is ignored if
     # log_config_append is set. (string value)
     # Deprecated group/name - [DEFAULT]/logfile
-    #log_file = /var/log/kuryr/cni.log
+    #log_file = <None>
 
     # (Optional) The base directory used for relative log_file  paths. This option
     # is ignored if log_config_append is set. (string value)
@@ -282,6 +396,12 @@ data:
     # is set. (boolean value)
     #use_syslog = false
 
+    # Enable journald for logging. If running in a systemd environment you may wish
+    # to enable journal support. Doing so will use the journal native protocol
+    # which includes structured metadata in addition to log messages.This option is
+    # ignored if log_config_append is set. (boolean value)
+    #use_journal = false
+
     # Syslog facility to receive log lines. This option is ignored if
     # log_config_append is set. (string value)
     #syslog_log_facility = LOG_USER
@@ -310,7 +430,7 @@ data:
 
     # List of package logging levels in logger=LEVEL pairs. This option is ignored
     # if log_config_append is set. (list value)
-    #default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO
+    #default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO
 
     # Enables or disables publication of error events. (boolean value)
     #publish_errors = false
@@ -323,14 +443,85 @@ data:
     # value)
     #instance_uuid_format = "[instance: %(uuid)s] "
 
+    # Interval, number of seconds, of log rate limiting. (integer value)
+    #rate_limit_interval = 0
+
+    # Maximum number of logged messages per rate_limit_interval. (integer value)
+    #rate_limit_burst = 0
+
+    # Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG
+    # or empty string. Logs with level greater or equal to rate_limit_except_level
+    # are not filtered. An empty string means that all levels are filtered. (string
+    # value)
+    #rate_limit_except_level = CRITICAL
+
     # Enables or disables fatal status of deprecations. (boolean value)
     #fatal_deprecations = false
 
 
     [binding]
+    # Configuration options for container interface binding.
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # The name prefix of the veth endpoint put inside the container. (string value)
+    #veth_dst_prefix = eth
+
+    # Driver to use for binding and unbinding ports. (string value)
+    # Deprecated group/name - [binding]/driver
+    #default_driver = kuryr.lib.binding.drivers.veth
+
+    # Drivers to use for binding and unbinding ports. (list value)
+    #enabled_drivers = kuryr.lib.binding.drivers.veth
+
+    # Specifies the name of the Nova instance interface to link the virtual devices
+    # to (only applicable to some binding drivers. (string value)
+    link_iface = eth0
 
     driver = kuryr.lib.binding.drivers.vlan
-    link_iface = {{ kuryr_cni_link_interface }}
+
+
+    [cni_daemon]
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # Enable CNI Daemon configuration. (boolean value)
+    daemon_enabled = true
+
+    # Bind address for CNI daemon HTTP server. It is recommened to allow only local
+    # connections. (string value)
+    bind_address = 127.0.0.1:50036
+
+    # Maximum number of processes that will be spawned to process requests from CNI
+    # driver. (integer value)
+    #worker_num = 30
+
+    # Time (in seconds) the CNI daemon will wait for VIF annotation to appear in
+    # pod metadata before failing the CNI request. (integer value)
+    #vif_annotation_timeout = 120
+
+    # Kuryr uses pyroute2 library to manipulate networking interfaces. When
+    # processing a high number of Kuryr requests in parallel, it may take kernel
+    # more time to process all networking stack changes. This option allows to tune
+    # internal pyroute2 timeout. (integer value)
+    #pyroute2_timeout = 30
+
+    # Set to True when you are running kuryr-daemon inside a Docker container on
+    # Kubernetes host. E.g. as DaemonSet on Kubernetes cluster Kuryr is supposed to
+    # provide networking for. This mainly means thatkuryr-daemon will look for
+    # network namespaces in $netns_proc_dir instead of /proc. (boolean value)
+    docker_mode = true
+
+    # When docker_mode is set to True, this config option should be set to where
+    # host's /proc directory is mounted. Please note that mounting it is necessary
+    # to allow Kuryr-Kubernetes to move host interfaces between host network
+    # namespaces, which is essential for Kuryr to work. (string value)
+    netns_proc_dir = /host_proc
+
 
     [kubernetes]
 
@@ -341,12 +532,136 @@ data:
     # The root URL of the Kubernetes API (string value)
     api_root = {{ openshift.master.api_url }}
 
-    # The token to talk to the k8s API
-    token_file = /etc/kuryr/token
+    # Absolute path to client cert to connect to HTTPS K8S_API (string value)
+    # ssl_client_crt_file = /etc/kuryr/controller.crt
+
+    # Absolute path client key file to connect to HTTPS K8S_API (string value)
+    # ssl_client_key_file = /etc/kuryr/controller.key
 
     # Absolute path to ca cert file to connect to HTTPS K8S_API (string value)
-    ssl_ca_crt_file = /etc/kuryr/ca.crt
+    ssl_ca_crt_file = /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+
+    # The token to talk to the k8s API
+    token_file = /var/run/secrets/kubernetes.io/serviceaccount/token
 
     # HTTPS K8S_API server identity verification (boolean value)
     # TODO (apuimedo): Make configurable
     ssl_verify_server_crt = True
+
+    # The driver to determine OpenStack project for pod ports (string value)
+    pod_project_driver = default
+
+    # The driver to determine OpenStack project for services (string value)
+    service_project_driver = default
+
+    # The driver to determine Neutron subnets for pod ports (string value)
+    pod_subnets_driver = default
+
+    # The driver to determine Neutron subnets for services (string value)
+    service_subnets_driver = default
+
+    # The driver to determine Neutron security groups for pods (string value)
+    pod_security_groups_driver = default
+
+    # The driver to determine Neutron security groups for services (string value)
+    service_security_groups_driver = default
+
+    # The driver that provides VIFs for Kubernetes Pods. (string value)
+    pod_vif_driver = nested-vlan
+
+    # The driver that manages VIFs pools for Kubernetes Pods (string value)
+    vif_pool_driver = {{ kuryr_openstack_enable_pools | default(False) | ternary('nested', 'noop') }}
+
+    [neutron]
+    # Configuration options for OpenStack Neutron
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # Authentication URL (string value)
+    auth_url = {{ kuryr_openstack_auth_url }}
+
+    # Authentication type to load (string value)
+    # Deprecated group/name - [neutron]/auth_plugin
+    auth_type = password
+
+    # Domain ID to scope to (string value)
+    user_domain_name = {{ kuryr_openstack_user_domain_name }}
+
+    # User's password (string value)
+    password = {{ kuryr_openstack_password }}
+
+    # Domain name containing project (string value)
+    project_domain_name = {{ kuryr_openstack_project_domain_name }}
+
+    # Project ID to scope to (string value)
+    # Deprecated group/name - [neutron]/tenant-id
+    project_id = {{ kuryr_openstack_project_id }}
+
+    # Token (string value)
+    #token = <None>
+
+    # Trust ID (string value)
+    #trust_id = <None>
+
+    # User's domain id (string value)
+    #user_domain_id = <None>
+
+    # User id (string value)
+    #user_id = <None>
+
+    # Username (string value)
+    # Deprecated group/name - [neutron]/user-name
+    username = {{kuryr_openstack_username }}
+
+    # Whether a plugging operation is failed if the port to plug does not become
+    # active (boolean value)
+    #vif_plugging_is_fatal = false
+
+    # Seconds to wait for port to become active (integer value)
+    #vif_plugging_timeout = 0
+
+    [neutron_defaults]
+
+    pod_security_groups = {{ kuryr_openstack_pod_sg_id }}
+    pod_subnet = {{ kuryr_openstack_pod_subnet_id }}
+    service_subnet = {{ kuryr_openstack_service_subnet_id }}
+    project = {{ kuryr_openstack_pod_project_id }}
+    # TODO (apuimedo): Remove the duplicated line just after this one once the
+    # RDO packaging contains the upstream patch
+    worker_nodes_subnet = {{ kuryr_openstack_worker_nodes_subnet_id }}
+
+    [pod_vif_nested]
+
+    worker_nodes_subnet = {{ kuryr_openstack_worker_nodes_subnet_id }}
+
+
+    [pool_manager]
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # Absolute path to socket file that will be used for communication with the
+    # Pool Manager daemon (string value)
+    #sock_file = /run/kuryr/kuryr_manage.sock
+
+
+    [vif_pool]
+
+    #
+    # From kuryr_kubernetes
+    #
+
+    # Set a maximun amount of ports per pool. 0 to disable (integer value)
+    ports_pool_max = {{ kuryr_openstack_pool_max | default(0) }}
+
+    # Set a target minimum size of the pool of ports (integer value)
+    ports_pool_min = {{ kuryr_openstack_pool_min | default(1) }}
+
+    # Number of ports to be created in a bulk request (integer value)
+    ports_pool_batch = {{ kuryr_openstack_pool_batch | default(5) }}
+
+    # Minimun interval (in seconds) between pool updates (integer value)
+    ports_pool_update_frequency = {{ kuryr_openstack_pool_update_frequency | default(20) }}