AWS ECR hosted docker containers. Local building of docker container.

.gitignore

@@ -1,3 +1,8 @@
 clairctl
 clairctl.yml
 reports/
+clairctl.zip
+vendor/
+.idea/
+*.iml
+.DS_Store

LocalDockerfile

@@ -0,0 +1,58 @@
+FROM alpine:3.5
+
+# Purpose: LocalDockerfile - allows building the clairctl from the local source files if you have cloned the project.
+# Dockerfile - downloads the source from github.com as a tar file and builds the clairctl via that source.
+
+ENV GOPATH=/go
+ENV PATH=${GOPATH}/bin:${PATH}
+ENV DOCKER_API_VERSION=1.24
+ARG DOCKER_VERSION=${DOCKER_VERSION:-latest}
+ARG CLAIRCTL_VERSION=${CLAIRCTL_VERSION:-master}
+ARG CLAIRCTL_COMMIT=
+
+RUN mkdir -p ${GOPATH}/src/github.com/jgsqware/clairctl/
+
+COPY clairctl.zip ${GOPATH}/src/github.com/jgsqware/clairctl/clairctl.zip
+
+RUN unzip ${GOPATH}/src/github.com/jgsqware/clairctl/clairctl.zip -d ${GOPATH}/src/github.com/jgsqware/clairctl/ \
+  && rm ${GOPATH}/src/github.com/jgsqware/clairctl/clairctl.zip
+
+RUN apk add --update curl \
+ && apk add --virtual build-dependencies go gcc build-base glide git \
+ && adduser clairctl -D \
+ && mkdir -p /reports \
+ && chown -R clairctl:clairctl /reports /tmp \
+ && curl https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz -o docker.tgz \
+ && tar xfvz docker.tgz --strip 1 -C /usr/bin/ docker/docker \
+ && rm -f docker.tgz \
+ && go get -u github.com/jteeuwen/go-bindata/... \
+ && cd ${GOPATH}/src/github.com/jgsqware/clairctl \
+ && glide install -v \
+ && go generate ./clair \
+ && go build -o /usr/local/bin/clairctl -ldflags "-X github.com/jgsqware/clairctl/cmd.version=${CLAIRCTL_VERSION}-${CLAIRCTL_COMMIT}" \
+ && apk del build-dependencies \
+ && rm -rf /var/cache/apk/* \
+ && rm -rf /root/.glide/ \
+ && rm -rf /go \
+ && echo $'clair:\n\
+  port: 6060\n\
+  healthPort: 6061\n\
+  uri: http://clair\n\
+  priority: Low\n\
+  report:\n\
+    path: /reports\n\
+    format: html\n\
+clairctl:\n\
+  port: 44480\n\
+  tempfolder: /tmp'\
+    > /home/clairctl/clairctl.yml
+
+USER clairctl
+
+WORKDIR /home/clairctl/
+
+EXPOSE 44480
+
+VOLUME ["/tmp/", "/reports/"]
+
+CMD ["/usr/sbin/crond", "-f"]

README.md

@@ -119,6 +119,130 @@ images:
     CVE-2016-7068: Something
 ```
 
+
+# Amazon AWS ECR and clairctl
+
+## Setup your environment for AWS ECR 
+
+### via .aws/credentials
+
+(Optional) If you choose to use your `~/.aws/credentials` file for configuration make the following changes.
+
+Your `~/.aws/credentials` you wwill have to add a section for each ECR `registry id` that you use. 
+
+For the below example the `registry id ` is `111111111111`
+
+Copy the `~/.aws/credentials` for `[default]` settings to create settings for `[111111111111]`
+
+E.G.:
+
+```bash
+
+[deafult]
+
+aws_access_key_id = YOUR_ACCESS_KEY_ID 
+aws_secret_access_key = YOUR_SECRET_ACCESS_KEY
+
+[111111111111]
+
+aws_access_key_id = YOUR_ACCESS_KEY_ID 
+aws_secret_access_key = YOUR_SECRET_ACCESS_KEY
+
+```
+
+### Via Environmental variables 
+
+#### Session_token and Access_key_id
+
+``` 
+    export AWS_SESSION_TOKEN=<token_value>
+    export AWS_ACCESS_KEY_ID=<key_value>
+```
+
+#### (OR) Secret_access_Key
+
+```
+   export AWS_SECRET_ACCESS_KEY=<secret_value>`
+```
+
+## Running Clairctl
+
+### Docker-Compose
+
+You can use the `docker-compose.yml` file that you can use to help start the 3 containers needed up.
+
+#### AWS configuration
+
+##### Set AWS_REGION from environmental variable 
+
+<https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html>
+
+replace `amazon-zone` with the zone for your ECR
+
+`export AWS_REGION=amazon-zone # e.g.: us-east-1 or eu-west-2`
+
+##### Set AWS_REGION in docker-compose.yml
+
+Uncomment the following entry in the `clairctl:` `environment:` of and add the value of your ECR region. 
+
+`#    - AWS_REGION= # put your region E.G.: us-east-1, eu-west-2 `
+
+E.G:
+
+`     - AWS_REGION=us-east-1 # put your region E.G.: us-east-1, eu-west-2 `
+
+##### Pass AWS Secrets to docker container 
+
+You will have to choose one of 3 options on passing the AWS secrets to the docker container.
+
+Uncomment the one type you chose.
+
+1. Use `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` from environmental variables.
+
+1. Use `AWS_SESSION_TOKEN` from an environmental variable
+
+1. Use the mounting of the `.aws` directory from your home directory. 
+
+
+#### Run Docker-compose  
+
+Start the 3 container
+
+` docker-compose up -d`
+
+#### Run commands against the clairctl docker instance.
+
+```
+    docker-compose exec clairctl clairctl COMMAND AWS_ECR_URL/your-company-or-grouping/your-container:docker_version
+```
+
+E.G.:
+
+```
+    docker-compose exec clairctl clairctl pull 111111111111.dkr.ecr.amazon-zone.amazonaws.com/your-company-or-grouping/your-container:docker_version
+```
+
+### Command line 
+
+#### Set AWS_REGION for go command line 
+
+<https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html>
+
+replace `amazon-zone` with the zone for your ECR
+
+`export AWS_REGION=amazon-zone # e.g.: us-east-1 or eu-west-2`
+
+#### Setup your .aws/credentials
+
+[Link to setup your environment for ECR](#setup-your-environment-for-aws-ecr)
+
+#### Run the clarictl command
+
+```
+    ./clairctl pull 111111111111.dkr.ecr.amazon-zone.amazonaws.com/your-company-or-grouping/your-container:docker_version
+```
+
+
 # Building the latest binaries
 
 **clairctl** requires Go 1.8+.
@@ -140,6 +264,34 @@ go build
 
 This will result in a `clairctl` executable in the `$GOPATH/src/github.com/jgsqware/clairctl` folder.
 
+# Build the Docker Container from Source Locally
+
+If you are making modifications to the source code and want to test it locally there is another docker file `LocalDockerfile`
+
+The project `Dockerfile` downloads a zip file of the source from the project from github.
+
+`https://github.com/jgsqware/clairctl/archive/master.zip`
+
+You will never see your local changes persisted into the container if you use `docker build .`
+
+There is a script to help building from the source.  
+
+`./local-docker.sh OPTIONAL_TAG_NAME`
+
+E.G.:
+
+Build a tagged version
+
+`./local-docker.sh jgsqware/clairctl:1.2.9`
+
+or 
+
+Build an untagged version for local development.
+
+`./local-docker.sh`
+
+Make sure to change the tag for clairctl in your local docker-compose.yml if you have built a different tagged version.
+
 # FAQ
 
 ## I get 400 errors !

VERSION

@@ -1 +1 @@
-1.2.8
+1.2.9

docker-compose.yml

@@ -27,9 +27,14 @@ services:
     restart: unless-stopped
     environment: 
       - DOCKER_API_VERSION=1.24
+#     - AWS_REGION= # put your region E.G.: us-east-1, eu-west-2
+#     - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}  # (AWS credentials option 1.) Environmental variable for AWS_ACCESS_KEY_ID.
+#     - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} # (AWS credentials option 1.) Environmental variable for AWS_SECRET_ACCESS_KEY.
+#     - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} # (AWS credentials option 2.) session token if not using (1.) user/password or (3.) mount of .aws. Environmental variable for AWS_SESSION_TOKEN.
     volumes:
       - ./docker-compose-data/clairctl-reports/:/reports/:rw
       - /var/run/docker.sock:/var/run/docker.sock:ro
-    depends_on: 
+#     - ${HOME}/.aws:/root/.aws/  # (AWS credentials option 3.) mount AWS credentials from ~/.aws/ in your home folder
+    depends_on:
       clair: 
         condition: service_started

docker/dockerdist/dockerdist.go

@@ -17,12 +17,20 @@
 package dockerdist
 
 import (
+	"encoding/base64"
 	"errors"
+	"fmt"
 	"net/url"
+	"os"
 	"reflect"
+	"regexp"
 
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/coreos/pkg/capnslog"
 	distlib "github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema1"
@@ -36,14 +44,16 @@ import (
 	"github.com/docker/docker/reference"
 	"github.com/docker/docker/registry"
 	"github.com/opencontainers/go-digest"
-	"golang.org/x/net/context"
 	"github.com/spf13/viper"
+	"golang.org/x/net/context"
 )
 
 var log = capnslog.NewPackageLogger("github.com/jgsqware/clairctl", "dockerdist")
 
 var ErrTagNotFound = errors.New("this image or tag is not found")
 
+var ecrRegex, _ = regexp.Compile(`((?P<registryId>[a-zA-Z0-9][a-zA-Z0-9_-]*)\.dkr\.ecr\.(?P<region>[a-zA-Z0-9][a-zA-Z0-9_-]*)\.amazonaws\.com(\.cn)?)(/(?P<imageName>[a-zA-Z0-9/_-]+))?(:(?P<tag>[a-zA-Z0-9_\-/]+))?`)
+
 func isInsecureRegistry(registryHostname string) bool {
 	for _, r := range viper.GetStringSlice("docker.insecure-registries") {
 		if r == registryHostname {
@@ -181,9 +191,83 @@ func getDigest(ctx context.Context, repo distlib.Repository, image reference.Nam
 	return descriptor.Digest, nil
 }
 
+func getEcrCredentials(image string) (types.AuthConfig, error) {
+
+	names := ecrRegex.SubexpNames()
+	captures := ecrRegex.FindAllStringSubmatch(image, -1)[0]
+	namedCaptures := map[string]string{}
+
+	for i, n := range captures {
+		namedCaptures[names[i]] = n
+	}
+
+	log.Debugf("The ECR registry id is %s\n", namedCaptures["registryId"])
+
+	// configure aws client
+	sess := session.New()
+	region, err := getRegion(sess)
+	if err != nil {
+		return types.AuthConfig{}, fmt.Errorf("Error fetching AWS region: %s\n", err.Error())
+	}
+	svc := ecr.New(sess, aws.NewConfig().WithMaxRetries(10).WithRegion(region))
+	registryId := namedCaptures["registryId"]
+
+	// this lets us handle multiple registries
+	params := &ecr.GetAuthorizationTokenInput{
+		RegistryIds: []*string{&registryId},
+	}
+
+	// request the token
+	resp, err := svc.GetAuthorizationToken(params)
+	if err != nil {
+		return types.AuthConfig{}, fmt.Errorf("Error authorizing: %s\n", err.Error())
+	}
+
+	// extract base64 token
+	data, err := base64.StdEncoding.DecodeString(*resp.AuthorizationData[0].AuthorizationToken)
+	if err != nil {
+		return types.AuthConfig{}, fmt.Errorf("Error decoding autorization token: %s\n", err.Error())
+	}
+
+	// extract username and password
+	token := strings.SplitN(string(data), ":", 2)
+
+	authConfig := types.AuthConfig{
+		Username:      token[0],
+		Password:      token[1],
+		ServerAddress: captures[1],
+	}
+
+	log.Debugf("Successfully logged into ECR")
+	log.Debugf("- Username: %s", authConfig.Username)
+	log.Debugf("- ServerAddress: %s", authConfig.ServerAddress)
+
+	return authConfig, nil
+}
+
+// if AWS_REGION not set, infer from instance metadata
+func getRegion(sess *session.Session) (string, error) {
+	region, exists := os.LookupEnv("AWS_REGION")
+	if !exists {
+		ec2region, err := ec2metadata.New(sess).Region()
+		if err != nil {
+			return "", fmt.Errorf("AWS_REGION not set and unable to fetch region from instance metadata: %s\n", err.Error())
+		}
+		region = ec2region
+	}
+	return region, nil
+}
+
 // GetAuthCredentials returns the auth credentials (if any found) for the given repository, as found
 // in the user's docker config.
 func GetAuthCredentials(image string) (types.AuthConfig, error) {
+	// Check if this is an ECR image
+	ecrMatch := ecrRegex.MatchString(image)
+
+	if ecrMatch {
+		return getEcrCredentials(image)
+	}
+
 	// Lookup the index information for the name.
 	indexInfo, err := registry.ParseSearchIndexInfo(image)
 	if err != nil {