preparation for a demo + some important bugfixes (#279)

* demo prep

* outline

* intermediate

* fixed a bug

* fix

* recent fixes

* fixes

* recent changes

* minor

* minor

* added comments

* fixes

SECURITY.md

@@ -9,7 +9,7 @@ Note that **Aegis** consists of more than a single project,
 and during a release cut, all projects are signed and tagged
 with the same version.
 
-Afer **Aegis** hits a major 1.0.0. version, this will change
+After **Aegis** hits a major 1.0.0. version, this will change,
 and we will also have a support plan various major versions.
 
 ## Reporting a Vulnerability
@@ -19,6 +19,6 @@ Send your vulnerability reports to [[email protected]](mailto:[email protected]
 We don’t have an official turnover time, but if nobody gets back
 to you within a week please send another email.
 
-We take all vulnerability reports seriously and you will be notified 
+We take all vulnerability reports seriously, and you will be notified 
 if your report is accepted or declined, and what further actions we are going
 to take on it.

app/safe/internal/bootstrap/bootstrap.go

@@ -46,33 +46,41 @@ func Monitor(
 	timedOut <-chan bool,
 ) {
 	counter := 3
-	select {
-	case <-acquiredSvid:
-		log.InfoLn(correlationId, "Acquired identity.")
-		counter--
+	for {
 		if counter == 0 {
-			state.Initialize()
-			log.DebugLn(correlationId, "Creating readiness probe.")
-			go probe.CreateReadiness()
+			break
 		}
-	case <-updatedSecret:
-		log.InfoLn(correlationId, "Updated age key.")
-		counter--
-		if counter == 0 {
-			state.Initialize()
-			log.DebugLn(correlationId, "Creating readiness probe.")
-			go probe.CreateReadiness()
-		}
-	case <-serverStarted:
-		log.InfoLn(correlationId, "Server ready.")
-		counter--
-		if counter == 0 {
-			state.Initialize()
-			log.DebugLn(correlationId, "Creating readiness probe.")
-			go probe.CreateReadiness()
+		select {
+		case <-acquiredSvid:
+			log.InfoLn(correlationId, "Acquired identity.")
+			counter--
+			log.InfoLn(correlationId, "remaining:", counter)
+			if counter == 0 {
+				state.Initialize()
+				log.DebugLn(correlationId, "Creating readiness probe.")
+				go probe.CreateReadiness()
+			}
+		case <-updatedSecret:
+			log.InfoLn(correlationId, "Updated age key.")
+			counter--
+			log.InfoLn(correlationId, "remaining:", counter)
+			if counter == 0 {
+				state.Initialize()
+				log.DebugLn(correlationId, "Creating readiness probe.")
+				go probe.CreateReadiness()
+			}
+		case <-serverStarted:
+			log.InfoLn(correlationId, "Server ready.")
+			counter--
+			log.InfoLn(correlationId, "remaining:", counter)
+			if counter == 0 {
+				state.Initialize()
+				log.DebugLn(correlationId, "Creating readiness probe.")
+				go probe.CreateReadiness()
+			}
+		case <-timedOut:
+			log.FatalLn(correlationId, "Failed to acquire an identity in a timely manner.")
 		}
-	case <-timedOut:
-		log.FatalLn(correlationId, "Failed to acquire an identity in a timely manner.")
 	}
 }
 
@@ -111,7 +119,9 @@ func AcquireSource(
 		)
 	}
 
+	log.TraceLn(id, "Sending: Acquired SVID", len(acquiredSvid))
 	acquiredSvid <- true
+	log.TraceLn(id, "Sent: Acquired SVID", len(acquiredSvid))
 
 	return source
 }
@@ -142,6 +152,7 @@ func CreateCryptoKey(id *string, updatedSecret chan<- bool) {
 	if secret != state.BlankAgeKeyValue {
 		log.InfoLn(id, "Secret has been set in the cluster, will reuse it")
 		state.SetAgeKey(secret)
+		updatedSecret <- true
 		return
 	}
 

app/safe/internal/server/server.go

@@ -12,7 +12,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/shieldworks/aegis/app/safe/internal/server/handle"
 	"github.com/shieldworks/aegis/core/env"
-	"github.com/shieldworks/aegis/core/probe"
 	"github.com/shieldworks/aegis/core/validation"
 	"github.com/spiffe/go-spiffe/v2/spiffeid"
 	"github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig"
@@ -45,9 +44,6 @@ func Serve(source *workloadapi.X509Source, serverStarted chan<- bool) error {
 
 	serverStarted <- true
 
-	// Since server has started, we can enable the readiness probe.
-	go probe.CreateReadiness()
-
 	if err := server.ListenAndServeTLS("", ""); err != nil {
 		return errors.Wrap(err, "serve: failed to listen and serve")
 	}

app/safe/internal/state/secret-queue-delete.go

@@ -48,17 +48,22 @@ func processSecretDeleteQueue() {
 		// Get a secret to be removed from the disk.
 		secret := <-secretDeleteQueue
 
-		cid := secret.Meta.CorrelationId
+		if secret.Name == "" {
+			log.WarnLn(&id, "processSecretDeleteQueue: trying to delete an empty secret. "+
+				"Possibly picked a nil secret", len(secretQueue))
+			return
+		}
 
-		log.TraceLn(&cid, "processSecretDeleteQueue: picked a secret", len(secretQueue))
+		log.TraceLn(&id, "processSecretDeleteQueue: picked a secret", len(secretQueue))
 
 		// Remove secret from disk.
 		dataPath := path.Join(env.SafeDataPath(), secret.Name+".age")
+		log.TraceLn(&id, "processSecretDeleteQueue: removing secret from disk:", dataPath)
 		err := os.Remove(dataPath)
-		if !os.IsNotExist(err) {
-			log.WarnLn(&cid, "processSecretDeleteQueue: failed to remove secret", err.Error())
+		if err != nil && !os.IsNotExist(err) {
+			log.WarnLn(&id, "processSecretDeleteQueue: failed to remove secret", err.Error())
 		}
 
-		log.TraceLn(&cid, "processSecretDeleteQueue: should have persisted the secret.")
+		log.TraceLn(&id, "processSecretDeleteQueue: should have deleted the secret.")
 	}
 }

app/safe/internal/state/state.go

@@ -225,10 +225,6 @@ func DeleteSecret(secret entity.SecretStored) {
 	s, exists := secrets.Load(secret.Name)
 	if !exists {
 		log.WarnLn(&cid, "DeleteSecret: Secret does not exist. Cannot delete.", secret.Name)
-
-		ss := s.(entity.SecretStored)
-		secret.Created = ss.Created
-
 		return
 	}
 

core/entity/data/v1/v1.go

@@ -218,8 +218,11 @@ func transform(secret SecretStored, value string) (string, error) {
 
 	switch secret.Meta.Format {
 	case None:
+		// Return the parsed string as is, without any further validation.
 		return parsedString, nil
 	case Json:
+		// If the parsed string is a valid JSON, return it as is.
+		// Otherwise, assume the parsing failed and return the original JSON string.
 		if tpl.ValidJSON(parsedString) {
 			return parsedString, nil
 		} else {
@@ -233,37 +236,37 @@ func transform(secret SecretStored, value string) (string, error) {
 			}
 			return yml, nil
 		} else {
-			yml, err := tpl.JsonToYaml(jsonData)
-			if err != nil {
-				return jsonData, err
-			}
-			return yml, nil
+			// Parsed string is not a valid JSON, so return it as is.
+			// It can be either a valid YAML already, or some random string.
+			// There is not much can be done at this point other than returning it.
+			return parsedString, nil
 		}
 	default:
-		return "", fmt.Errorf("unknown format: %s", secret.Meta.Format)
+		// The program flow shall never enter here.
+		return parsedString, fmt.Errorf("unknown format: %s", secret.Meta.Format)
 	}
 }
 
 // Parse takes a data.SecretStored type as input and returns the parsed
 // string or an error.
 //
-// If the Meta.Template field is empty, it tries to parse the first secret.Values;
-// otherwise it transforms secret.Values[0] using the Go template transformation
-// defined by Meta.Template.
-//
-// If the Meta.Format field is None, it returns the parsed string.
-//
-// If the Meta.Format field is Json, it returns the parsed string if it’s a
-// valid JSON or the original string otherwise.
+// It parses all the `.Values` of the secret, and for each value tries to apply
+// a template transformation.
 //
-// If the Meta.Format field is Yaml, it tries its best to transform the data
-// into Yaml. If it fails, it tries to return a valid JSON at least. If that
-// fails too, returns the original secret value.
+// Here is how the template transformation is applied:
 //
-// If the Meta.Format field is not recognized, it returns an empty string.
+//  1. Compute parsedString:
+//     If the Meta.Template field is empty, then parsedString is the original value.
+//     Otherwise, parsedString is the result of applying the template transformation
+//     to the original value.
 //
-// If there is more than one value in the Values collection then the transformation
-// is applied to each value and the result is returned as a JSON array.
+// 2.	Compute the output string:
+//   - If the Meta.Format field is None, then the output string is parsedString.
+//   - If the Meta.Format field is Json, then the output string is parsedString
+//     if parsedString is a valid JSON, otherwise it’s the original value.
+//   - If the Meta.Format field is Yaml, then the output string is the result of
+//     transforming parsedString into Yaml if parsedString is a valid JSON,
+//     otherwise it’s parsedString.
 func (secret SecretStored) Parse() (string, error) {
 	if len(secret.Values) == 0 {
 		return "", fmt.Errorf("no values found for secret %s", secret.Name)

core/template/template.go

@@ -60,16 +60,22 @@ func JsonToYaml(js string) (string, error) {
 //
 // On successful execution, the function returns the resulting string from the
 // executed template.
-func TryParse(tmpStr, json string) string {
+func TryParse(tmpStr, jason string) string {
 	tmpl, err := template.New("secret").Parse(tmpStr)
 	if err != nil {
-		return json
+		return jason
+	}
+
+	var result map[string]any
+	err = json.Unmarshal([]byte(jason), &result)
+	if err != nil {
+		return jason
 	}
 
 	var tpl bytes.Buffer
-	err = tmpl.Execute(&tpl, json)
+	err = tmpl.Execute(&tpl, result)
 	if err != nil {
-		return json
+		return jason
 	}
 
 	return tpl.String()

examples/aegis-workshop/README.md

@@ -0,0 +1,12 @@
+# Aegis
+
+![Aegis](../../assets/aegis-icon.png "Aegis")
+
+## Aegis Workshop
+
+This workshop is designed to help you get started with Aegis. It will walk you 
+through the installation of Aegis and its components, and then demonstrate how 
+to use Aegis to protect your secrets.
+
+There is a work-in-progress video recording of this workshop. We’ll update this
+document with a link to the video once it’s ready.

examples/aegis-workshop/delete-secret.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+#
+# .-'_.---._'-.
+# ||####|(__)||   Protect your secrets, protect your business.
+#   \\()|##//       Secure your sensitive data with Aegis.
+#    \\ |#//                    <aegis.ist>
+#     .\_/.
+#
+
+. ./env.sh
+
+# FIXME: -s argument should not be needed.
+kubectl exec "$SENTINEL" -n aegis-system -- aegis \
+-w "aegis-workload-demo" \
+-s "dummy" \
+-d

examples/aegis-workshop/delete-workload.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+#
+# .-'_.---._'-.
+# ||####|(__)||   Protect your secrets, protect your business.
+#   \\()|##//       Secure your sensitive data with Aegis.
+#    \\ |#//                    <aegis.ist>
+#     .\_/.
+#
+
+. ./env.sh
+
+kubectl delete deployment "$DEPLOYMENT" -n default

examples/aegis-workshop/encrypt-secret.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+#
+# .-'_.---._'-.
+# ||####|(__)||   Protect your secrets, protect your business.
+#   \\()|##//       Secure your sensitive data with Aegis.
+#    \\ |#//                    <aegis.ist>
+#     .\_/.
+#
+
+. /home/v0lk4n/Desktop/AEGIS/aegis/examples/aegis-workshop/env.sh
+
+kubectl exec "$SENTINEL" -n aegis-system -- aegis \
+-s '{"username": "root", "password": "SuperSecret", "value": "AegisRocks"}' \
+-e

examples/aegis-workshop/env.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+#
+# .-'_.---._'-.
+# ||####|(__)||   Protect your secrets, protect your business.
+#   \\()|##//       Secure your sensitive data with Aegis.
+#    \\ |#//                    <aegis.ist>
+#     .\_/.
+#
+
+
+export SECRET="ComputeMe!"
+
+export SENTINEL=$(kubectl get po -n aegis-system \
+  | grep "aegis-sentinel-" | awk '{print $1}')
+
+export SAFE=$(kubectl get po -n aegis-system \
+  | grep "aegis-safe-" | awk '{print $1}')
+
+export WORKLOAD=$(kubectl get po -n default \
+  | grep "aegis-workload-demo-" | awk '{print $1}')
+
+export INSPECTOR=$(kubectl get po -n default \
+  | grep "aegis-inspector-" | awk '{print $1}')
+
+export DEPLOYMENT="aegis-workload-demo"

examples/aegis-workshop/ids/Inspector.yaml

@@ -0,0 +1,26 @@
+#
+# .-'_.---._'-.
+# ||####|(__)||   Protect your secrets, protect your business.
+#   \\()|##//       Secure your sensitive data with Aegis.
+#    \\ |#//                    <aegis.ist>
+#     .\_/.
+#
+
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterSPIFFEID
+metadata:
+  name: aegis-inspector
+spec:
+  # SPIFFE ID `MUST` start with "spiffe://aegis.ist/workload/$workloadName/ns/"
+  # for `aegis-safe` to recognize the workload and dispatch secrets to it.
+  spiffeIDTemplate: "spiffe://aegis.ist\
+    /workload/aegis-workload-demo\
+    /ns/default\
+    /sa/aegis-workload-demo\
+    /n/{{ .PodMeta.Name }}"
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: aegis-inspector
+  workloadSelectorTemplates:
+  - "k8s:ns:default"
+  - "k8s:sa:aegis-inspector"

examples/aegis-workshop/ids/Workload.yaml

@@ -0,0 +1,26 @@
+#
+# .-'_.---._'-.
+# ||####|(__)||   Protect your secrets, protect your business.
+#   \\()|##//       Secure your sensitive data with Aegis.
+#    \\ |#//                    <aegis.ist>
+#     .\_/.
+#
+
+apiVersion: spire.spiffe.io/v1alpha1
+kind: ClusterSPIFFEID
+metadata:
+  name: aegis-workload-demo
+spec:
+  # SPIFFE ID `MUST` start with "spiffe://aegis.ist/workload/$workloadName/ns/"
+  # for `aegis-safe` to recognize the workload and dispatch secrets to it.
+  spiffeIDTemplate: "spiffe://aegis.ist\
+    /workload/aegis-workload-demo\
+    /ns/{{ .PodMeta.Namespace }}\
+    /sa/{{ .PodSpec.ServiceAccountName }}\
+    /n/{{ .PodMeta.Name }}"
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: aegis-workload-demo
+  workloadSelectorTemplates:
+  - "k8s:ns:default"
+  - "k8s:sa:aegis-workload-demo"