From fb2bfb4c49e352c29fc25ef700de7784d30bb6e4 Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Sun, 24 Sep 2023 17:24:13 +0200 Subject: [PATCH] NPEP: Iron out Egress Support API Design Signed-off-by: Surya Seetharaman --- npep/npep-126-egress-traffic-control.md | 175 +++++++++++++++++++++++- 1 file changed, 170 insertions(+), 5 deletions(-) diff --git a/npep/npep-126-egress-traffic-control.md b/npep/npep-126-egress-traffic-control.md index 530df79b..f7c4dd8f 100644 --- a/npep/npep-126-egress-traffic-control.md +++ b/npep/npep-126-egress-traffic-control.md @@ -1,7 +1,7 @@ # NPEP-126: Add northbound traffic support in (B)ANP API * Issue: [#126](https://github.com/kubernetes-sigs/network-policy-api/issues/126) -* Status: Provisional +* Status: Implementable ## TLDR @@ -76,13 +76,178 @@ selected cluster workloads to k8s-apiservers for securing the server. ## API -(... details, can point to PR with changes) - +Proof of Concept for the API design details can be found here: https://github.com/kubernetes-sigs/network-policy-api/pull/143 + +### Implementing egress traffic control towards cluster nodes + +This NPEP proposes to add a new type of `AdminNetworkPolicyEgressPeer` called `Nodes` +to be able to explicitly select nodes (based on the node's labels) in the cluster. +This ensures that if the list of IPs on a node OR list of nodes change, the users +don't need to manually intervene to include those new IPs. The label selectors will +take care of this automatically. Note that the nodeIPs that this type of peer matches +on are the IPs present in `Node.Status.Addresses` field of the node. + +``` +// AdminNetworkPolicyEgressPeer defines a peer to allow traffic to. +// Exactly one of the selector pointers must be set for a given peer. If a +// consumer observes none of its fields are set, they must assume an unknown +// option has been specified and fail closed. +// +kubebuilder:validation:MaxProperties=1 +// +kubebuilder:validation:MinProperties=1 +type AdminNetworkPolicyEgressPeer struct { + + // Nodes defines a way to select a set of nodes in + // in the cluster. This field follows standard label selector + // semantics; if present but empty, it selects all Nodes. + // + // Support: Core + // + // +optional + Nodes *metav1.LabelSelector `json:"nodes,omitempty"` +} +``` + +Note that `AdminNetworkPolicyPeer` will be changed to +`AdminNetworkPolicyEgressPeer` and `AdminNetworkPolicyIngressPeer` since ingress and +egress peers have started to diverge at this point and it is easy to +maintain it with two sets of peer definitions. +This ensures nodes can be referred to only as "egress peers". + +Example: Admin wants to deny egress traffic from tenants who don't have +`restricted`, `confidential` or `internal` level security clearance +to control-plane nodes at 443 and 6443 ports in the cluster + +``` +apiVersion: policy.networking.k8s.io/v1alpha1 +kind: AdminNetworkPolicy +metadata: + name: node-as-egress-peers +spec: + priority: 55 + subject: + namespaces: + matchExpressions: + - {key: security, operator: notIn, values: [restricted, confidential, internal]} + egress: + - name: "deny-all-egress-to-kapi-server" + action: "Deny" + to: + - nodes: + matchLabels: + node-role.kubernetes.io/control-plane: + ports: + - portNumber: + protocol: TCP + port: 443 + - portNumber: + protocol: TCP + port: 6443 +``` + +### Implementing egress traffic control towards external destinations + +This NPEP proposes to add a new type of `AdminNetworkPolicyEgressPeer` called `ExternalNetworks` +to be able to explicitly select external destination CIDRs outside the cluster. +This peer type will not be supported in `AdminNetworkPolicyIngressPeer`. + +``` +// AdminNetworkPolicyEgressPeer defines a peer to allow traffic to. +// Exactly one of the selector pointers must be set for a given peer. If a +// consumer observes none of its fields are set, they must assume an unknown +// option has been specified and fail closed. +// +kubebuilder:validation:MaxProperties=1 +// +kubebuilder:validation:MinProperties=1 +type AdminNetworkPolicyEgressPeer struct { + + // ExternalNetworks defines a way to select ExternalNetworkSets + // that consist of network CIDRs that live outside the cluster as a peer. + // It is the list of NetworkCIDR (both v4 & v6) that can be used to define + // external destinations. + // This field follows standard label selector semantics; if present + // but empty, it selects all ExternalNetworkSets defined in the cluster. + // + // Support: Core + // + // +optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=100 + ExternalNetworks []string `json:"externalNetworks,omitempty" validate:"omitempty,dive,cidr"` +} +``` + +Note that the API expects `externalNetworks` to be a set of "external" destinations. +However if user puts a podCIDR, nodeCIDR, serviceCIDR or other intra-cluster +networks, it is implementation-defined as to what the behavior will be. Not +all implementations can correctly define the boundary between "internal" and +"external" destinations with respect to a Kubernetes cluster. + +Example: -> after we reach consensus on externalNetwork versus network + +Let's define ANP and BANP that refer to these external networks: +``` +apiVersion: policy.networking.k8s.io/v1alpha1 +kind: AdminNetworkPolicy +metadata: + name: cluster-egress-controls +spec: + priority: 70 + subject: + namespaces: {} + egress: + - name: "allow-all-egress-to-intranet" + action: "Allow" + to: + - externalNetworks: + matchLabels: + network: intranet + - name: "allow-egress-to-external-dns" + action: "Allow" + to: + - externalNetworks: + matchLabels: + network: dns + ports: + - portNumber: + protocol: UDP + port: 53 + - name: "pass-egress-to-internet" + action: "Pass" + to: + - externalNetworks: + matchLabels: + network: internet +--- +apiVersion: policy.networking.k8s.io/v1alpha1 +kind: BaselineAdminNetworkPolicy +metadata: + name: default +spec: + subject: + namespaces: {} + egress: + - name: "deny-egress-to-internet" + action: "Deny" + to: + - externalNetworks: + matchLabels: + network: internet +``` +This allows admins to specify all cluster workloads can talk to +the company's internet CIDR's and the external DNS network. It +also allows them to put up default guardrails of not allowing +any other egress traffic towards internet in a BANP. ## Alternatives -(List other design alternatives and why we did not go in that -direction) +* Instead of adding CIDR peer directly into the main object, we can +define a new object called `ExternalNetworkSet` and use selectors or +name of that object to be referred to from AdminNetworkPolicy and +BaselineAdminNetworkPolicy objects. This is particularly useful +if CIDR ranges are prone to changes versus the current model is +is better if the set of CIDRs are mostly a constant and are only referred +to from one or two egress rules. It increases readability. However the +drawback is if the CIDRs do change, then one has to ensure to update all +the relevant ANPs and BANP accordingly. ## References