poc of kubernetes entity provider #51
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| #!/usr/bin/env node | ||
| require('../build/lib/providers/kubernetes/start') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| FROM node:18-alpine | ||
| MAINTAINER Roadie | ||
| WORKDIR /usr/src/app | ||
| COPY package.json . | ||
| RUN npm install && npm install typescript -g | ||
| COPY . . | ||
| RUN yarn build | ||
| CMD ["node", "./bin/kubernetes-entity-provider"] | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,7 +8,7 @@ RUN node --version | |
| RUN apt-get update && apt-get install -y ca-certificates | ||
| ENV NPM_CONFIG_PREFIX=/home/node/.npm-global | ||
| ENV PATH=$PATH:/home/node/.npm-global/bin | ||
| RUN npm install --global snyk-broker@4.157.5 | ||
|
|
||
| # Removing [email protected] (setheader transitive) to satisfy some scanners still reporting this false positive | ||
| RUN rm -rf /home/node/.npm-global/lib/node_modules/snyk-broker/node_modules/setheader/node_modules/debug | ||
|
|
||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| import {EntityProviderHandler} from "$/types"; | ||
| import {ComponentEntity, Entity, SystemEntity} from "@backstage/catalog-model"; | ||
| import {isError} from "@backstage/errors"; | ||
| import {getRootLogger} from "@/logger"; | ||
| import {KubeConfig, CoreV1Api, V1Namespace, V1ServiceList, CustomObjectsApi} from '@kubernetes/client-node'; | ||
| import {renderString} from 'nunjucks'; | ||
| import yaml from 'yaml' | ||
|
|
||
| const ROADIE_OWNER_ANNOTATION = 'roadie.io/owner' | ||
| const ROADIE_LIFECYCLE_ANNOTATION = 'roadie.io/lifecycle' | ||
|
|
||
| type KubernetesEntityProviderOptions = { | ||
| namespaceFilters?: { | ||
| labelSelector?: string | ||
| fieldSelector?: string | ||
| }, | ||
| objectMappings: Array<{ | ||
| group?: string, | ||
| version?: string, | ||
| plural: string, | ||
| labelSelector?: string, | ||
| fieldSelector?: string, | ||
| template: string | ||
| }> | ||
| } | ||
|
|
||
| export class KubernetesEntityProvider { | ||
| private readonly opts: KubernetesEntityProviderOptions; | ||
| private readonly kc: KubeConfig; | ||
|
|
||
| constructor(opts: KubernetesEntityProviderOptions) { | ||
| this.opts = opts | ||
| this.kc = new KubeConfig(); | ||
| this.kc.loadFromDefault() | ||
| } | ||
|
|
||
| private async listNamespaces() { | ||
| const k8sApi = this.kc.makeApiClient(CoreV1Api); | ||
| const {body: namespaces} = await k8sApi.listNamespace(undefined, undefined, undefined, this.opts.namespaceFilters?.fieldSelector, this.opts.namespaceFilters?.labelSelector); | ||
| return namespaces | ||
| } | ||
|
|
||
| private async listPods(namespace: string, labelSelector?: string, fieldSelector?: string) { | ||
| const k8sApi = this.kc.makeApiClient(CoreV1Api); | ||
| const {body: pods} = await k8sApi.listNamespacedPod(namespace, undefined, undefined, undefined, fieldSelector, labelSelector); | ||
| return pods | ||
| } | ||
|
|
||
| handler: EntityProviderHandler = async (emit) => { | ||
| const logger = getRootLogger(); | ||
| logger.info(`handling entity request`) | ||
| try { | ||
| const entities: Entity[] = [] | ||
|
|
||
| const k8sCustomObjectsApi = this.kc.makeApiClient(CustomObjectsApi) | ||
| const namespaces = await this.listNamespaces(); | ||
|
|
||
| await Promise.all(namespaces.items.map(async namespace => { | ||
|
There was a problem hiding this comment. I think flatMapping should probably achieve the same things here, depending how you decide to modify the loop below |
||
| await Promise.all(this.opts.objectMappings.map(async objectMapping => { | ||
|
There was a problem hiding this comment. await for...of probably easier to read here since you are mutating the external var already |
||
| if (namespace.metadata?.name) { | ||
| if (objectMapping.plural === 'pods') { | ||
|
There was a problem hiding this comment. For the core api, I am only allow pods to be collected, in the ultimate version we would allow collecting any of the other core api resources. |
||
| const list = await this.listPods(namespace.metadata?.name, objectMapping.labelSelector, objectMapping.fieldSelector); | ||
| list.items.forEach(pod => { | ||
| const entityString = renderString(objectMapping.template, pod); | ||
| const items = yaml.parseAllDocuments(entityString).map(doc => doc.toJSON()) | ||
| entities.push(...items) | ||
| }) | ||
| } | ||
|
|
||
| if (objectMapping.group && objectMapping.version) { | ||
| const { response } = await k8sCustomObjectsApi.listNamespacedCustomObject( | ||
| objectMapping.group, | ||
| objectMapping.version, | ||
| namespace.metadata.name, | ||
| objectMapping.plural | ||
| ); | ||
| (response as any).body.items.forEach((customResource: any) => { | ||
| const entityString = renderString(objectMapping.template, customResource); | ||
| const items = yaml.parseAllDocuments(entityString).map(doc => doc.toJSON()) | ||
| entities.push(...items) | ||
| }) | ||
| } | ||
| } | ||
| })) | ||
| })); | ||
|
|
||
| await emit({ | ||
| type: "full", | ||
| entities: entities.map(entity => ({ | ||
| entity, | ||
| locationKey: 'test' | ||
| })) | ||
| }) | ||
| } catch (e: any) { | ||
| logger.info(`handling entity request failed: ${e}`); | ||
|
There was a problem hiding this comment. We probably need better error handling here. |
||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| { | ||
| "private": [ | ||
| { | ||
| "method": "GET", | ||
| "path": "/agent-provider/*", | ||
| "origin": "http://localhost:7342" | ||
| } | ||
| ], | ||
| "public": [ | ||
| { | ||
| "method": "any", | ||
| "path": "/*" | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| #!/usr/bin/env ts-node | ||
| import {createRoadieAgentEntityProvider, RoadieAgent} from "../../"; | ||
| import {KubernetesEntityProvider} from "@/providers/kubernetes/EntityProvider"; | ||
| import {getRootLogger} from "@/logger"; | ||
| import yaml from 'yaml'; | ||
|
|
||
| const podTemplate = ` | ||
| --- | ||
| apiVersion: backstage.io/v1alpha1 | ||
| kind: Component | ||
| metadata: | ||
| name: {{ metadata.name }} | ||
| namespace: {{ metadata.namespace }} | ||
| spec: | ||
| type: service | ||
| lifecycle: unknown | ||
| owner: test | ||
| `; | ||
|
|
||
| const helmChartTemplate = ` | ||
| --- | ||
| apiVersion: backstage.io/v1alpha1 | ||
| kind: Resource | ||
| metadata: | ||
| name: {{ metadata.name }} | ||
| namespace: {{ metadata.namespace }} | ||
| spec: | ||
| type: 'helm-chart' | ||
| lifecycle: unknown | ||
| owner: test | ||
| `; | ||
|
|
||
| const main = async () => { | ||
| const logger = getRootLogger(); | ||
| const brokerServer = process.env.BROKER_SERVER || 'http://localhost:7341'; | ||
|
|
||
| RoadieAgent.fromConfig({ | ||
| server: brokerServer, | ||
|
There was a problem hiding this comment. The settings here will need to be provided from the outside.. and some of the settings should be defaulted in the implementation of fromConfig. |
||
| port: 7342, | ||
| identifier: 'kubernetes-entity-agent', | ||
| accept: '/Users/brianfletcher/git-repos/roadie-agent/config/accept.json', | ||
| agentPort: 7044 | ||
| }) | ||
| .addEntityProvider( | ||
| createRoadieAgentEntityProvider({ | ||
| name: 'kube', | ||
| handler: new KubernetesEntityProvider({ | ||
| namespaceFilters: { | ||
|
There was a problem hiding this comment. All of these settings should be configurable with sensible defaults. |
||
|
|
||
| }, | ||
| objectMappings: [{ | ||
| template: podTemplate, | ||
| plural: 'pods', | ||
| }, | ||
| { | ||
| template: helmChartTemplate, | ||
| version: 'v1beta1', | ||
| group: 'source.toolkit.fluxcd.io', | ||
| plural: 'helmcharts', | ||
| }]}).handler | ||
| }), | ||
| ) | ||
| .start(); | ||
| } | ||
|
|
||
| void (async () => { | ||
| try { | ||
| await main() | ||
| } catch(err) { | ||
| console.error('Something bad') | ||
| } | ||
| })() | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
need the ability to configure the entity provider entity mappings and the broker client server url and token.