From bc1a76066c0aeb43776ded0b266ec48f5e69aa16 Mon Sep 17 00:00:00 2001 From: Jayc Date: Tue, 31 Oct 2023 20:29:55 +0530 Subject: [PATCH 1/2] feat: add support to add custom network policies for specific workspaces in faas pods --- src/util/customTransformer-faas.js | 16 ++++++-- src/util/openfaas/index.js | 62 +++++++++++++++++++++++++----- src/v0/util/index.js | 7 ++++ 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js index d1fa48d2d1..54d2410313 100644 --- a/src/util/customTransformer-faas.js +++ b/src/util/customTransformer-faas.js @@ -1,7 +1,7 @@ const { v4: uuidv4 } = require('uuid'); const crypto = require('crypto'); const NodeCache = require('node-cache'); -const { getMetadata } = require('../v0/util'); +const { getMetadata, getTransformationMetadata } = require('../v0/util'); const stats = require('./stats'); const { setupFaasFunction, @@ -82,10 +82,10 @@ async function setOpenFaasUserTransform( libraryVersionIds, pregeneratedFnName, testMode = false, + trMetadata = {}, ) { const tags = { transformerVersionId: userTransformation.versionId, - language: userTransformation.language, identifier: 'openfaas', testMode, }; @@ -106,6 +106,7 @@ async function setOpenFaasUserTransform( testMode, ), testMode, + trMetadata, ); stats.timing('creation_time', setupTime, tags); @@ -129,16 +130,22 @@ async function runOpenFaasUserTransform( const metaTags = events[0].metadata ? getMetadata(events[0].metadata) : {}; const tags = { transformerVersionId: userTransformation.versionId, - language: userTransformation.language, identifier: 'openfaas', testMode, ...metaTags, }; + const trMetadata = events[0].metadata ? getTransformationMetadata(events[0].metadata) : {}; // check and deploy faas function if not exists const functionName = generateFunctionName(userTransformation, libraryVersionIds, testMode); if (testMode) { - await setOpenFaasUserTransform(userTransformation, libraryVersionIds, functionName, testMode); + await setOpenFaasUserTransform( + userTransformation, + libraryVersionIds, + functionName, + testMode, + trMetadata, + ); } const invokeTime = new Date(); @@ -156,6 +163,7 @@ async function runOpenFaasUserTransform( testMode, ), testMode, + trMetadata, ); stats.timing('run_time', invokeTime, tags); return result; diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js index 60ad316e1b..f80aa01c23 100644 --- a/src/util/openfaas/index.js +++ b/src/util/openfaas/index.js @@ -23,6 +23,8 @@ const CONFIG_BACKEND_URL = process.env.CONFIG_BACKEND_URL || 'https://api.rudder const GEOLOCATION_URL = process.env.GEOLOCATION_URL || ''; const FAAS_AST_VID = 'ast'; const FAAS_AST_FN_NAME = 'fn-ast'; +const CUSTOM_NETWORK_POLICY_WORKSPACE_IDS = process.env.CUSTOM_NETWORK_POLICY_WORKSPACE_IDS || ''; +const customNetworkPolicyWorkspaceIds = CUSTOM_NETWORK_POLICY_WORKSPACE_IDS.split(','); // Initialise node cache const functionListCache = new NodeCache(); @@ -111,7 +113,14 @@ const invalidateFnCache = () => { functionListCache.set(FUNC_LIST_KEY, []); }; -const deployFaasFunction = async (functionName, code, versionId, libraryVersionIDs, testMode) => { +const deployFaasFunction = async ( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata = {}, +) => { try { logger.debug('[Faas] Deploying a faas function'); let envProcess = 'python index.py'; @@ -132,6 +141,22 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI if (GEOLOCATION_URL) { envVars.geolocation_url = GEOLOCATION_URL; } + // labels + const labels = { + 'openfaas-fn': 'true', + 'parent-component': 'openfaas', + 'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT, + 'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT, + transformationId: trMetadata.transformationId, + workspaceId: trMetadata.workspaceId, + }; + if ( + trMetadata.workspaceId && + customNetworkPolicyWorkspaceIds.includes(trMetadata.workspaceId) + ) { + labels['custom-network-policy'] = 'true'; + } + // TODO: investigate and add more required labels and annotations const payload = { service: functionName, @@ -139,12 +164,7 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI image: FAAS_BASE_IMG, envProcess, envVars, - labels: { - 'openfaas-fn': 'true', - 'parent-component': 'openfaas', - 'com.openfaas.scale.max': FAAS_MAX_PODS_IN_TEXT, - 'com.openfaas.scale.min': FAAS_MIN_PODS_IN_TEXT, - }, + labels, annotations: { 'prometheus.io.scrape': 'true', }, @@ -175,14 +195,28 @@ const deployFaasFunction = async (functionName, code, versionId, libraryVersionI } }; -async function setupFaasFunction(functionName, code, versionId, libraryVersionIDs, testMode) { +async function setupFaasFunction( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata = {}, +) { try { if (!testMode && isFunctionDeployed(functionName)) { logger.debug(`[Faas] Function ${functionName} already deployed`); return; } // deploy faas function - await deployFaasFunction(functionName, code, versionId, libraryVersionIDs, testMode); + await deployFaasFunction( + functionName, + code, + versionId, + libraryVersionIDs, + testMode, + trMetadata, + ); // This api call is only used to check if function is spinned correctly await awaitFunctionReadiness(functionName); @@ -201,6 +235,7 @@ const executeFaasFunction = async ( versionId, libraryVersionIDs, testMode, + trMetadata = {}, ) => { try { logger.debug('[Faas] Invoking faas function'); @@ -217,7 +252,14 @@ const executeFaasFunction = async ( error.message.includes(`error finding function ${functionName}`) ) { removeFunctionFromCache(functionName); - await setupFaasFunction(functionName, null, versionId, libraryVersionIDs, testMode); + await setupFaasFunction( + functionName, + null, + versionId, + libraryVersionIDs, + testMode, + trMetadata, + ); throw new RetryRequestError(`${functionName} not found`); } diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 4ea3d3783d..bed515624c 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -1403,6 +1403,12 @@ const getMetadata = (metadata) => ({ destinationType: metadata.destinationType, k8_namespace: metadata.namespace, }); + +const getTransformationMetadata = (metadata) => ({ + transformationId: metadata.transformationId, + workspaceId: metadata.workspaceId, +}); + // checks if array 2 is a subset of array 1 function checkSubsetOfArray(array1, array2) { const result = array2.every((val) => array1.includes(val)); @@ -2113,6 +2119,7 @@ module.exports = { getIntegrationsObj, getMappingConfig, getMetadata, + getTransformationMetadata, getParsedIP, getStringValueOfJSON, getSuccessRespEvents, From 3e1fb26422f4e274a48df129e74600374669a0a6 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Thu, 2 Nov 2023 23:50:11 +0000 Subject: [PATCH 2/2] chore(release): 1.48.0 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7d295645f..01631435d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.48.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.47.0...v1.48.0) (2023-11-02) + + +### Features + +* add support to add custom network policies for specific workspaces in faas pods ([bc1a760](https://github.com/rudderlabs/rudder-transformer/commit/bc1a76066c0aeb43776ded0b266ec48f5e69aa16)) + ## [1.47.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.46.5...v1.47.0) (2023-10-30) diff --git a/package-lock.json b/package-lock.json index 4643d1325d..0822a9b42b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.47.0", + "version": "1.48.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.47.0", + "version": "1.48.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "^0.7.24", diff --git a/package.json b/package.json index adc5f0e8f5..46f728664d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.47.0", + "version": "1.48.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": {