update(notebook-controller): kf1.9 upgrade

components/notebook-controller/Makefile

@@ -98,7 +98,7 @@ docker-push: ## Push docker image with the manager.
 
 .PHONY: docker-build-multi-arch
 docker-build-multi-arch: ##  Build multi-arch docker images with docker buildx
-	cd .. && docker buildx build --platform ${ARCH} --tag ${IMG}:${TAG} -f ./notebook-controller/Dockerfile .
+	cd .. && docker buildx build --load --platform ${ARCH} --tag ${IMG}:${TAG} -f ./notebook-controller/Dockerfile .
 
 
 .PHONY: docker-build-push-multi-arch
@@ -144,20 +144,18 @@ kustomize: ## Download kustomize locally if necessary.
 	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/v3/cmd/[email protected])
 
 ENVTEST = $(shell pwd)/bin/setup-envtest
+ENVTEST_VERSION?=release-0.14
+
 .PHONY: envtest
 envtest: ## Download envtest-setup locally if necessary.
-	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
+	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION))
 
 # go-get-tool will 'go get' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 define go-get-tool
 @[ -f $(1) ] || { \
 set -e ;\
-TMP_DIR=$$(mktemp -d) ;\
-cd $$TMP_DIR ;\
-go mod init tmp ;\
 echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
-rm -rf $$TMP_DIR ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
 }
 endef

components/notebook-controller/OWNERS

@@ -1,8 +1,7 @@
 approvers:
-  - elikatsis
   - kimwnasptd
-  - StefanoFioravanzo
   - thesuperzapper
+emeritus_approvers:
+  - elikatsis
+  - StefanoFioravanzo
   - yanniszark
-reviewers:
-  - MartinForReal

components/notebook-controller/README.md

@@ -22,18 +22,6 @@ spec:
       containers:
         - name: my-notebook
           image: kubeflownotebookswg/jupyter:master
-          args:
-            [
-              "start.sh",
-              "lab",
-              "--LabApp.token=''",
-              "--LabApp.allow_remote_access='True'",
-              "--LabApp.allow_root='True'",
-              "--LabApp.ip='*'",
-              "--LabApp.base_url=/test/my-notebook/",
-              "--port=8888",
-              "--no-browser",
-            ]
 ```
 
 The required fields are `containers[0].image` and (`containers[0].command` and/or `containers[0].args`).

components/notebook-controller/config/base/kustomization.yaml

@@ -5,4 +5,4 @@ resources:
 images:
 - name: docker.io/kubeflownotebookswg/notebook-controller
   newName: docker.io/kubeflownotebookswg/notebook-controller
-  newTag: v1.7.0
+  newTag: v1.9.2

components/notebook-controller/config/crd/kustomization.yaml

@@ -21,3 +21,12 @@ patchesStrategicMerge:
 # the following config is for teaching kustomize how to do kustomization for CRDs.
 configurations:
 - kustomizeconfig.yaml
+
+patchesJson6902:
+  - target:
+      group: apiextensions.k8s.io
+      version: v1
+      kind: CustomResourceDefinition
+      name: notebooks.kubeflow.org
+    path: patches/validation_patches.yaml
+

components/notebook-controller/config/crd/patches/validation_patches.yaml

@@ -0,0 +1,29 @@
+- op: replace
+  path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/containers/items/required
+  value: 
+    - name
+    - image
+
+- op: replace
+  path: /spec/versions/1/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/containers/items/required
+  value: 
+    - name
+    - image
+
+- op: replace
+  path: /spec/versions/2/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/containers/items/required
+  value: 
+    - name
+    - image
+
+- op: add
+  path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/containers/minItems
+  value: 1
+
+- op: add
+  path: /spec/versions/1/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/containers/minItems
+  value: 1
+
+- op: add
+  path: /spec/versions/2/schema/openAPIV3Schema/properties/spec/properties/template/properties/spec/properties/containers/minItems
+  value: 1

components/notebook-controller/config/manager/manager.yaml

@@ -32,6 +32,11 @@ spec:
               configMapKeyRef:
                 name: config
                 key: ISTIO_GATEWAY
+          - name: ISTIO_HOST
+            valueFrom:
+              configMapKeyRef:
+                name: config
+                key: ISTIO_HOST
           - name: CLUSTER_DOMAIN
             valueFrom:
               configMapKeyRef:

components/notebook-controller/config/manager/params.env

@@ -1,6 +1,7 @@
 USE_ISTIO=true
 ISTIO_GATEWAY=kubeflow/kubeflow-gateway
+ISTIO_HOST=*
 CLUSTER_DOMAIN=cluster.local
 ENABLE_CULLING=false
 CULL_IDLE_TIME=1440
-IDLENESS_CHECK_PERIOD=1
+IDLENESS_CHECK_PERIOD=1

components/notebook-controller/controllers/notebook_controller.go

@@ -210,7 +210,7 @@ func (r *NotebookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	foundPod := &corev1.Pod{}
 	err = r.Get(ctx, types.NamespacedName{Name: ss.Name + "-0", Namespace: ss.Namespace}, foundPod)
 	if err != nil && apierrs.IsNotFound(err) {
-		log.Info(fmt.Sprintf("No Pods are currently running for Notebook Server: %s in namesace: %s.", instance.Name, instance.Namespace))
+		log.Info(fmt.Sprintf("No Pods are currently running for Notebook Server: %s in namespace: %s.", instance.Name, instance.Namespace))
 	} else if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -377,20 +377,32 @@ func generateStatefulSet(instance *v1beta1.Notebook) *appsv1.StatefulSet {
 				},
 			},
 			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{
-					"statefulset":   instance.Name,
-					"notebook-name": instance.Name,
-				}},
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"statefulset":   instance.Name,
+						"notebook-name": instance.Name,
+					},
+					Annotations: map[string]string{},
+				},
 				Spec: *instance.Spec.Template.Spec.DeepCopy(),
 			},
 		},
 	}
+
 	// copy all of the Notebook labels to the pod including poddefault related labels
 	l := &ss.Spec.Template.ObjectMeta.Labels
 	for k, v := range instance.ObjectMeta.Labels {
 		(*l)[k] = v
 	}
 
+	// copy all of the Notebook annotations to the pod.
+	a := &ss.Spec.Template.ObjectMeta.Annotations
+	for k, v := range instance.ObjectMeta.Annotations {
+		if !strings.Contains(k, "kubectl") && !strings.Contains(k, "notebook") {
+			(*a)[k] = v
+		}
+	}
+
 	podSpec := &ss.Spec.Template.Spec
 	container := &podSpec.Containers[0]
 	if container.WorkingDir == "" {
@@ -484,8 +496,14 @@ func generateVirtualService(instance *v1beta1.Notebook) (*unstructured.Unstructu
 	vsvc.SetKind("VirtualService")
 	vsvc.SetName(virtualServiceName(name, namespace))
 	vsvc.SetNamespace(namespace)
-	if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{"*"}, "spec", "hosts"); err != nil {
-		return nil, fmt.Errorf("set .spec.hosts error: %v", err)
+
+	istioHost := os.Getenv("ISTIO_HOST")
+	if len(istioHost) == 0 {
+		istioHost = "*"
+	}
+	if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{istioHost}, "spec", "hosts"); err != nil {
+		return nil, fmt.Errorf("Set .spec.hosts error: %v", err)
+
 	}
 
 	istioGateway := os.Getenv("ISTIO_GATEWAY")

components/notebook-controller/go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/kubeflow/kubeflow/components/common v0.0.0-20220218084159-4ad0158e955e
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.11.1
 	k8s.io/api v0.23.0
 	k8s.io/apimachinery v0.23.0
 	k8s.io/client-go v0.23.0
@@ -63,7 +63,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
 	k8s.io/apiextensions-apiserver v0.23.0 // indirect
 	k8s.io/component-base v0.23.0 // indirect
 	k8s.io/klog/v2 v2.30.0 // indirect

components/notebook-controller/go.sum

@@ -386,8 +386,9 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -912,8 +913,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

components/notebook-controller/loadtest/start_notebooks.py

@@ -1,8 +1,8 @@
 # This script aims to load test Kubeflow Notebook controller by starting
 # certain nubmer of Kubeflow Notebook custom resources.
 #
-# Before the test, make sure you have connected to your desired Kubeflow cluster
-# and have enough Kubernetes resources (or have autoscaling turned on).
+# Before the test, make sure you have connected to your desired Kubeflow
+# cluster and have enough Kubernetes resources (or have autoscaling turned on).
 #
 # To start the load test, you can run
 #   python3.8 start_notebooks.py -l <#notebooks> -n <namespace>
@@ -12,6 +12,7 @@
 
 import argparse
 import subprocess
+
 import yaml
 
 parser = argparse.ArgumentParser(
@@ -48,48 +49,51 @@
 
 
 def write_notebook_config(config, name, num):
-  config['metadata']['name'] = 'jupyter-test-' + str(num)
-  config['spec']['template']['spec']['containers'][0]['name'
-                                                     ] = 'notebook-' + str(num)
-  config['spec']['template']['spec']['volumes'][0]['persistentVolumeClaim'][
-      'claimName'] = 'test-vol-' + str(num)
-  with open(name, 'w') as f:
-    print(yaml.dump(config), file=f)
+    config['metadata']['name'] = 'jupyter-test-' + str(num)
+    spec = config['spec']['template']['spec']
+    spec['containers'][0]['name'] = 'notebook-' + str(num)
+    pvc = spec['volumes'][0]['persistentVolumeClaim']
+    pvc['claimName'] = 'test-vol-' + str(num)
+    with open(name, 'w') as f:
+        print(yaml.dump(config), file=f)
 
 
 def write_pvc_config(config, name, num):
-  config['metadata']['name'] = 'test-vol-' + str(num)
-  with open(name, 'w') as f:
-    print(yaml.dump(config), file=f)
+    config['metadata']['name'] = 'test-vol-' + str(num)
+    with open(name, 'w') as f:
+        print(yaml.dump(config), file=f)
 
 
 def main():
-  args = parser.parse_args()
-  assert args.operation == 'apply' or args.operation == 'delete'
-  notebook_config = None
-  pvc_config = None
-  with open('jupyter_test.yaml', 'r') as f:
-    notebook_config = yaml.safe_load(f.read())
-  with open('jupyter_pvc.yaml', 'r') as f:
-    pvc_config = yaml.safe_load(f.read())
-  for i in range(args.num_notebooks):
-    notebook_name = f'jupyter_test_{i}.yaml'
-    pvc_name = f'jupyter_pvc_{i}.yaml'
-    write_notebook_config(notebook_config, notebook_name, i)
-    write_pvc_config(pvc_config, pvc_name, i)
-    print(f'kubectl {args.operation} -f {notebook_name} ...')
-    subprocess.run([
-        'kubectl', args.operation, '-f', notebook_name, '-n', args.namespace
-    ],
-                   capture_output=True,
-                   check=True)
-    print(f'kubectl {args.operation} -f {pvc_name} ...')
-    subprocess.run([
-        'kubectl', args.operation, '-f', pvc_name, '-n', args.namespace
-    ],
-                   capture_output=True,
-                   check=True)
+    args = parser.parse_args()
+    assert args.operation == 'apply' or args.operation == 'delete'
+    notebook_config = None
+    pvc_config = None
+    with open('jupyter_test.yaml', 'r') as f:
+        notebook_config = yaml.safe_load(f.read())
+    with open('jupyter_pvc.yaml', 'r') as f:
+        pvc_config = yaml.safe_load(f.read())
+    for i in range(args.num_notebooks):
+        notebook_name = f'jupyter_test_{i}.yaml'
+        pvc_name = f'jupyter_pvc_{i}.yaml'
+        write_notebook_config(notebook_config, notebook_name, i)
+        write_pvc_config(pvc_config, pvc_name, i)
+        print(f'kubectl {args.operation} -f {notebook_name} ...')
+        subprocess.run(['kubectl',
+                        args.operation,
+                        '-f',
+                        notebook_name,
+                        '-n',
+                        args.namespace],
+                       capture_output=True,
+                       check=True)
+        print(f'kubectl {args.operation} -f {pvc_name} ...')
+        subprocess.run([
+            'kubectl', args.operation, '-f', pvc_name, '-n', args.namespace
+        ],
+            capture_output=True,
+            check=True)
 
 
 if __name__ == '__main__':
-  main()
+    main()