From 840c051e12ce52f188c86fb74788fe651f218d8e Mon Sep 17 00:00:00 2001
From: Alexander Litvinenko <altvnk@me.com>
Date: Wed, 19 Aug 2015 11:37:30 +0300
Subject: [PATCH 1/4] Add kube-proxy to master node. This makes portal-ip's
 reachable from master node.

---
 roles/kubernetes/tasks/gen_tokens.yml      |  2 +-
 roles/master/handlers/main.yml             |  7 ++++++
 roles/master/tasks/main.yml                | 27 ++++++++++++++++++++++
 roles/master/templates/proxy.j2            |  7 ++++++
 roles/master/templates/proxy.kubeconfig.j2 | 18 +++++++++++++++
 5 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 roles/master/templates/proxy.j2
 create mode 100644 roles/master/templates/proxy.kubeconfig.j2

diff --git a/roles/kubernetes/tasks/gen_tokens.yml b/roles/kubernetes/tasks/gen_tokens.yml
index 821418c..2ce702d 100644
--- a/roles/kubernetes/tasks/gen_tokens.yml
+++ b/roles/kubernetes/tasks/gen_tokens.yml
@@ -12,7 +12,7 @@
   environment:
     TOKEN_DIR: "{{ kube_token_dir }}"
   with_nested:
-    - [ "system:controller_manager", "system:scheduler", "system:kubectl" ]
+    - [ "system:controller_manager", "system:scheduler", "system:kubectl", 'system:proxy' ]
     - "{{ groups[master_group_name] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
diff --git a/roles/master/handlers/main.yml b/roles/master/handlers/main.yml
index 2b50991..84b1e53 100644
--- a/roles/master/handlers/main.yml
+++ b/roles/master/handlers/main.yml
@@ -6,6 +6,7 @@
     - restart apiserver
     - restart controller-manager
     - restart scheduler
+    - restart proxy
 
 - name: restart apiserver
   sudo: yes
@@ -24,3 +25,9 @@
   service:
     name: kube-scheduler
     state: restarted
+
+- name: restart proxy
+  sudo: yes
+  service:
+    name: kube-proxy
+    state: restarted
\ No newline at end of file
diff --git a/roles/master/tasks/main.yml b/roles/master/tasks/main.yml
index 53d481a..403cb2d 100644
--- a/roles/master/tasks/main.yml
+++ b/roles/master/tasks/main.yml
@@ -10,6 +10,17 @@
   tags:
     - master
 
+- name: install kubernetes node
+  sudo: yes
+  yum:
+    pkg=kubernetes-node
+    state=latest
+    enablerepo=virt7-docker-common-candidate
+  notify:
+    - restart daemons
+  tags:
+    - master
+
 - name: get the node token values from token files
   sudo: yes
   slurp:
@@ -18,6 +29,7 @@
     - "system:controller_manager"
     - "system:scheduler"
     - "system:kubectl"
+    - "system:proxy"
   register: tokens
   delegate_to: "{{ groups[master_group_name][0] }}"
   tags:
@@ -28,6 +40,7 @@
     controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
     scheduler_token: "{{ tokens.results[1].content|b64decode }}"
     kubectl_token: "{{ tokens.results[2].content|b64decode }}"
+    proxy_token: "{{ tokens.results[3].content|b64decode }}"
   tags:
     - master
 
@@ -77,6 +90,20 @@
   tags:
     - master
 
+- name: write the config files for proxy
+  sudo: yes
+  template: src=proxy.j2 dest={{ kube_config_dir }}/proxy
+  notify:
+    - restart daemons
+  tags:
+    - master
+
+- name: write the kubecfg (auth) file for proxy
+  sudo: yes
+  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig
+  tags:
+    - master
+
 - name: populate users for basic auth in API
   sudo: yes
   lineinfile:
diff --git a/roles/master/templates/proxy.j2 b/roles/master/templates/proxy.j2
new file mode 100644
index 0000000..1a1f7b1
--- /dev/null
+++ b/roles/master/templates/proxy.j2
@@ -0,0 +1,7 @@
+###
+# kubernetes proxy config
+
+# default config should be adequate
+
+# Add your own!
+KUBE_PROXY_ARGS="--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig"
diff --git a/roles/master/templates/proxy.kubeconfig.j2 b/roles/master/templates/proxy.kubeconfig.j2
new file mode 100644
index 0000000..43aba40
--- /dev/null
+++ b/roles/master/templates/proxy.kubeconfig.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+current-context: proxy-to-{{ cluster_name }}
+preferences: {}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: proxy
+  name: proxy-to-{{ cluster_name }}
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: https://{{ groups[master_group_name][0] }}:{{ kube_master_port }}
+  name: {{ cluster_name }}
+users:
+- name: proxy
+  user:
+    token: {{ proxy_token }}

From 0a776ddb28bce3ef723f8c8e560292255ca7d105 Mon Sep 17 00:00:00 2001
From: Alexander Litvinenko <altvnk@me.com>
Date: Wed, 19 Aug 2015 16:16:48 +0300
Subject: [PATCH 2/4] Add dnsmasq role

---
 roles/dnsmasq/handlers/main.yml             |  8 ++
 roles/dnsmasq/tasks/main.yml                | 98 +++++++++++++++++++++
 roles/dnsmasq/templates/01-kube-dns.conf.j2 | 13 +++
 roles/dnsmasq/templates/resolv.conf.j2      |  8 ++
 setup.yml                                   |  2 +
 5 files changed, 129 insertions(+)
 create mode 100644 roles/dnsmasq/handlers/main.yml
 create mode 100644 roles/dnsmasq/tasks/main.yml
 create mode 100644 roles/dnsmasq/templates/01-kube-dns.conf.j2
 create mode 100644 roles/dnsmasq/templates/resolv.conf.j2

diff --git a/roles/dnsmasq/handlers/main.yml b/roles/dnsmasq/handlers/main.yml
new file mode 100644
index 0000000..07cc484
--- /dev/null
+++ b/roles/dnsmasq/handlers/main.yml
@@ -0,0 +1,8 @@
+---
+- name: restart networkmanager
+  sudo: yes
+  command: systemctl restart NetworkManager
+
+- name: restart dnsmasq
+  sudo: yes
+  command: systemctl restart dnsmasq
\ No newline at end of file
diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml
new file mode 100644
index 0000000..ef17037
--- /dev/null
+++ b/roles/dnsmasq/tasks/main.yml
@@ -0,0 +1,98 @@
+---
+- name: install dnsmasq and bind-utils
+  sudo: yes
+  yum:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - dnsmasq
+    - bind-utils
+  when: inventory_hostname in groups[master_group_name]
+  tags:
+    - dnsmasq
+
+- name: install networkmanager
+  sudo: yes
+  yum:
+    name: NetworkManager
+    state: latest
+  tags:
+    - dnsmasq
+
+- name: collect nameservers
+  sudo: yes
+  shell: "cat /etc/resolv.conf | grep -i '^nameserver' | cut -d ' ' -f2"
+  register: nameservers_output
+  tags:
+    - dnsmasq
+
+- name: collect dns search list
+  sudo: yes
+  shell: "cat /etc/resolv.conf | grep -i '^search' | cut -d ' ' -f2- | tr ' ' '\n'"
+  register: dns_search_list_output
+  tags:
+    - dnsmasq
+
+- name: set nameservers
+  set_fact:
+    nameservers_list: "{{ nameservers_output.stdout_lines }}"
+  tags:
+    - dnsmasq
+
+- name: set dns search list
+  set_fact:
+    domain_search_list: "{{ dns_search_list_output.stdout_lines }}"
+  tags:
+    - dnsmasq
+
+- name: ensure dnsmasq.d directory exists
+  sudo: yes
+  file:
+    path: /etc/NetworkManager/dnsmasq.d
+    state: directory
+  when: inventory_hostname in groups[master_group_name]
+  tags:
+    - dnsmasq
+
+- name: configure dnsmasq
+  sudo: yes
+  template:
+    src: 01-kube-dns.conf.j2
+    dest: /etc/dnsmasq.d/01-kube-dns.conf
+    mode: 755
+  notify:
+    - restart dnsmasq
+  when: inventory_hostname in groups[master_group_name]
+  tags:
+    - dnsmasq
+
+- name: enable dnsmasq
+  sudo: yes
+  service:
+    name: dnsmasq
+    state: started
+    enabled: yes
+  when: inventory_hostname in groups[master_group_name]
+  tags:
+    - dnsmasq
+
+- name: disable NetworkManager DNS config
+  sudo: yes
+  ini_file: 
+    dest: /etc/NetworkManager/NetworkManager.conf
+    section: main
+    option: dns
+    value: none
+  notify:
+    - restart networkmanager
+  tags: 
+    - dnsmasq
+
+- name: update resolv.conf with new DNS setup
+  sudo: yes
+  template:
+    src: resolv.conf.j2
+    dest: /etc/resolv.conf
+    mode: 644
+  tags:
+    - dnsmasq
\ No newline at end of file
diff --git a/roles/dnsmasq/templates/01-kube-dns.conf.j2 b/roles/dnsmasq/templates/01-kube-dns.conf.j2
new file mode 100644
index 0000000..0d4df73
--- /dev/null
+++ b/roles/dnsmasq/templates/01-kube-dns.conf.j2
@@ -0,0 +1,13 @@
+#Listen on all interfaces
+interface=*
+
+addn-hosts=/etc/hosts
+
+bogus-priv
+
+#Set upstream dns servers
+server=8.8.8.8
+server=8.8.4.4
+
+# Forward k8s domain to kube-dns
+server=/{{ dns_domain }}/{{ dns_server }}
\ No newline at end of file
diff --git a/roles/dnsmasq/templates/resolv.conf.j2 b/roles/dnsmasq/templates/resolv.conf.j2
new file mode 100644
index 0000000..22bc968
--- /dev/null
+++ b/roles/dnsmasq/templates/resolv.conf.j2
@@ -0,0 +1,8 @@
+; generated by ansible
+search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | union(domain_search_list) | unique | join(' ') }}
+{% for host in groups[master_group_name] %}
+nameserver {{ hostvars[host]['ansible_default_ipv4']['address'] }}
+{% endfor %}
+{% for nameserver in nameservers_list | unique %}
+nameserver {{ nameserver }}
+{% endfor %}
\ No newline at end of file
diff --git a/setup.yml b/setup.yml
index 7807774..289158d 100644
--- a/setup.yml
+++ b/setup.yml
@@ -13,6 +13,7 @@
     - flannel
     - master
     - addons
+    - dnsmasq
 
 # provide the execution plane
 - hosts: role=node
@@ -21,3 +22,4 @@
     - docker
     - flannel
     - minion
+    - dnsmasq

From 579f9cbc2333d42dadbd24187907de9c332f7889 Mon Sep 17 00:00:00 2001
From: Alexander Litvinenko <altvnk@me.com>
Date: Wed, 19 Aug 2015 16:45:03 +0300
Subject: [PATCH 3/4] Install meta package `kubernets`

---
 roles/master/tasks/main.yml | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/roles/master/tasks/main.yml b/roles/master/tasks/main.yml
index 403cb2d..3b6ed34 100644
--- a/roles/master/tasks/main.yml
+++ b/roles/master/tasks/main.yml
@@ -2,18 +2,7 @@
 - name: install kubernetes master
   sudo: yes
   yum:
-    pkg=kubernetes-master
-    state=latest
-    enablerepo=virt7-docker-common-candidate
-  notify:
-    - restart daemons
-  tags:
-    - master
-
-- name: install kubernetes node
-  sudo: yes
-  yum:
-    pkg=kubernetes-node
+    pkg=kubernetes
     state=latest
     enablerepo=virt7-docker-common-candidate
   notify:

From 9951bd888e3f93223d9f3c590f2143ead915e4c0 Mon Sep 17 00:00:00 2001
From: Alexander Litvinenko <altvnk@me.com>
Date: Fri, 21 Aug 2015 13:13:24 +0300
Subject: [PATCH 4/4] - Remove NetworkManager - Forget about DNS configuration
 provided by cloud - Disable /etc/resolv.conf modification by dhclient - Fix
 issue when kube-proxy is not started after reboot

---
 roles/dnsmasq/handlers/main.yml        |  4 --
 roles/dnsmasq/tasks/main.yml           | 58 +++++---------------------
 roles/dnsmasq/templates/resolv.conf.j2 |  5 +--
 roles/master/tasks/main.yml            |  9 ++++
 4 files changed, 21 insertions(+), 55 deletions(-)

diff --git a/roles/dnsmasq/handlers/main.yml b/roles/dnsmasq/handlers/main.yml
index 07cc484..f545f70 100644
--- a/roles/dnsmasq/handlers/main.yml
+++ b/roles/dnsmasq/handlers/main.yml
@@ -1,8 +1,4 @@
 ---
-- name: restart networkmanager
-  sudo: yes
-  command: systemctl restart NetworkManager
-
 - name: restart dnsmasq
   sudo: yes
   command: systemctl restart dnsmasq
\ No newline at end of file
diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml
index ef17037..e24e043 100644
--- a/roles/dnsmasq/tasks/main.yml
+++ b/roles/dnsmasq/tasks/main.yml
@@ -11,44 +11,10 @@
   tags:
     - dnsmasq
 
-- name: install networkmanager
-  sudo: yes
-  yum:
-    name: NetworkManager
-    state: latest
-  tags:
-    - dnsmasq
-
-- name: collect nameservers
-  sudo: yes
-  shell: "cat /etc/resolv.conf | grep -i '^nameserver' | cut -d ' ' -f2"
-  register: nameservers_output
-  tags:
-    - dnsmasq
-
-- name: collect dns search list
-  sudo: yes
-  shell: "cat /etc/resolv.conf | grep -i '^search' | cut -d ' ' -f2- | tr ' ' '\n'"
-  register: dns_search_list_output
-  tags:
-    - dnsmasq
-
-- name: set nameservers
-  set_fact:
-    nameservers_list: "{{ nameservers_output.stdout_lines }}"
-  tags:
-    - dnsmasq
-
-- name: set dns search list
-  set_fact:
-    domain_search_list: "{{ dns_search_list_output.stdout_lines }}"
-  tags:
-    - dnsmasq
-
 - name: ensure dnsmasq.d directory exists
   sudo: yes
   file:
-    path: /etc/NetworkManager/dnsmasq.d
+    path: /etc/dnsmasq.d
     state: directory
   when: inventory_hostname in groups[master_group_name]
   tags:
@@ -76,23 +42,21 @@
   tags:
     - dnsmasq
 
-- name: disable NetworkManager DNS config
-  sudo: yes
-  ini_file: 
-    dest: /etc/NetworkManager/NetworkManager.conf
-    section: main
-    option: dns
-    value: none
-  notify:
-    - restart networkmanager
-  tags: 
-    - dnsmasq
-
 - name: update resolv.conf with new DNS setup
   sudo: yes
   template:
     src: resolv.conf.j2
     dest: /etc/resolv.conf
     mode: 644
+  tags:
+    - dnsmasq
+
+- name: disable resolv.conf modification by dhclient
+  sudo: yes
+  lineinfile: 
+    dest: "/etc/sysconfig/network-scripts/ifcfg-{{ ansible_default_ipv4.interface }}"
+    state: present
+    regexp: '^PEERDNS'
+    line: 'PEERDNS="no"'
   tags:
     - dnsmasq
\ No newline at end of file
diff --git a/roles/dnsmasq/templates/resolv.conf.j2 b/roles/dnsmasq/templates/resolv.conf.j2
index 22bc968..c9ef825 100644
--- a/roles/dnsmasq/templates/resolv.conf.j2
+++ b/roles/dnsmasq/templates/resolv.conf.j2
@@ -1,8 +1,5 @@
 ; generated by ansible
-search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | union(domain_search_list) | unique | join(' ') }}
+search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | join(' ') }}
 {% for host in groups[master_group_name] %}
 nameserver {{ hostvars[host]['ansible_default_ipv4']['address'] }}
-{% endfor %}
-{% for nameserver in nameservers_list | unique %}
-nameserver {{ nameserver }}
 {% endfor %}
\ No newline at end of file
diff --git a/roles/master/tasks/main.yml b/roles/master/tasks/main.yml
index 3b6ed34..4a41c7e 100644
--- a/roles/master/tasks/main.yml
+++ b/roles/master/tasks/main.yml
@@ -129,5 +129,14 @@
     name: kube-scheduler
     enabled: yes
     state: started
+  tags:
+    - master
+
+- name: Enable kube-proxy
+  sudo: yes
+  service:
+    name: kube-proxy
+    enabled: yes
+    state: started
   tags:
     - master
\ No newline at end of file