Merge branch 'main' into pdf-upload-error

cli/magic-config.ts

@@ -72,6 +72,7 @@ const cfCountries = getCountryCodesAndNames();
 
 const iamRoleRegExp = RegExp(/arn:aws:iam::\d+:role\/[\w-_]+/);
 const kendraIdRegExp = RegExp(/^\w{8}-\w{4}-\w{4}-\w{4}-\w{12}$/);
+const secretManagerArnRegExp = RegExp(/arn:aws:secretsmanager:[\w-_]+:\d+:secret:[\w-_]+/);
 
 const embeddingModels = [
   {
@@ -144,6 +145,7 @@ const embeddingModels = [
       options.enableSagemakerModels = config.llms?.sagemaker
         ? config.llms?.sagemaker.length > 0
         : false;
+      options.huggingfaceApiSecretArn = config.llms?.huggingfaceApiSecretArn;
       options.enableSagemakerModelsSchedule = config.llms?.sagemakerSchedule?.enabled;
       options.timezonePicker = config.llms?.sagemakerSchedule?.timezonePicker;
       options.enableCronFormat = config.llms?.sagemakerSchedule?.enableCronFormat;
@@ -356,6 +358,22 @@ async function processCreateOptions(options: any): Promise<void> {
         return !(this as any).state.answers.enableSagemakerModels;
       },
     },
+    {
+      type: "input",
+      name: "huggingfaceApiSecretArn",
+      message:
+        "Some HuggingFace models including mistral now require an API key, Please enter an Secrets Manager Secret ARN (see docs: Model Requirements)",
+      validate: (v: string) => {
+        const valid = secretManagerArnRegExp.test(v);
+        return v.length === 0 || valid
+          ? true
+          : "If you are supplying a HF API key it needs to be a reference to a secrets manager secret ARN"
+      },
+      initial: options.huggingfaceApiSecretArn || "",
+      skip(): boolean {
+        return !(this as any).state.answers.enableSagemakerModels;
+      },
+    },
     {
       type: "confirm",
       name: "enableSagemakerModelsSchedule",
@@ -724,6 +742,7 @@ async function processCreateOptions(options: any): Promise<void> {
       : undefined,
     llms: {
       sagemaker: answers.sagemakerModels,
+      huggingfaceApiSecretArn: answers.huggingfaceApiSecretArn,
       sagemakerSchedule: answers.enableSagemakerModelsSchedule
         ? {
             enabled: answers.enableSagemakerModelsSchedule,

docs/documentation/model-requirements.md

@@ -19,6 +19,18 @@ Note:
 
 ![sample](./assets/enable-models.gif "AWS GenAI Chatbot")
 
+## HuggingFace Authentication
+
+Some models hosted on HuggingFace require an API key for access, for example MistralAI and Meta models have now changed to be gated behind accepting their EULA
+
+If you wish to continue using these models or access other models on HuggingFace which require authentication you can now supply this HF token as part of the installer.
+
+When enabling sagemaker models in the installer it will now ask you for a Secrets Manager Secret ARN containing the HF API token.
+
+You can read more about setting up access tokens on the [HF website](https://huggingface.co/docs/hub/en/security-tokens) Once you've got a token you may need to also navigate to a models page such as mistral7B to accept their terms before you can then use your token to access the model.
+
+The secret you would create in secrets manager would be a plain text secret containing just the HF token itself.
+
 ## Third-party models requirements
 
 You can also interact with external providers via their API, such as AI21 Labs, Cohere, OpenAI, etc.

lib/models/index.ts

@@ -2,6 +2,7 @@ import * as cdk from "aws-cdk-lib";
 import * as ssm from "aws-cdk-lib/aws-ssm";
 import { Construct } from "constructs";
 import * as iam from "aws-cdk-lib/aws-iam";
+import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
 import { Shared } from "../shared";
 import {
   Modality,
@@ -34,6 +35,10 @@ export class Models extends Construct {
 
     const models: SageMakerModelEndpoint[] = [];
 
+    let hfTokenSecret: secretsmanager.Secret | undefined;
+    if (props.config.llms.huggingfaceApiSecretArn) {
+      hfTokenSecret = secretsmanager.Secret.fromSecretCompleteArn(this, 'HFTokenSecret', props.config.llms.huggingfaceApiSecretArn) as secretsmanager.Secret;
+    }
     if (
       props.config.llms?.sagemaker.includes(SupportedSageMakerModels.FalconLite)
     ) {
@@ -106,11 +111,12 @@ export class Models extends Construct {
             ),
           },
           container:
-            DeepLearningContainerImage.HUGGINGFACE_PYTORCH_TGI_INFERENCE_2_0_1_TGI1_1_0_GPU_PY39_CU118_UBUNTU20_04,
+            DeepLearningContainerImage.HUGGINGFACE_PYTORCH_TGI_INFERENCE_2_1_1_TGI2_0_0_GPU_PY310_CU121_UBUNTU22_04,
           instanceType: SageMakerInstanceType.ML_G5_2XLARGE,
           startupHealthCheckTimeoutInSeconds: 300,
           endpointName: MISTRAL_7B_ENDPOINT_NAME,
           environment: {
+            HF_TOKEN: hfTokenSecret?.secretValue.unsafeUnwrap().toString() || "",
             SM_NUM_GPUS: JSON.stringify(1),
             MAX_INPUT_LENGTH: JSON.stringify(2048),
             MAX_TOTAL_TOKENS: JSON.stringify(4096),
@@ -152,14 +158,12 @@ export class Models extends Construct {
               (subnet) => subnet.subnetId
             ),
           },
-          container: DeepLearningContainerImage.fromDeepLearningContainerImage(
-            "huggingface-pytorch-tgi-inference",
-            "2.1.1-tgi1.3.3-gpu-py310-cu121-ubuntu20.04"
-          ),
+          container: DeepLearningContainerImage.HUGGINGFACE_PYTORCH_TGI_INFERENCE_2_1_1_TGI2_0_0_GPU_PY310_CU121_UBUNTU22_04,
           instanceType: SageMakerInstanceType.ML_G5_2XLARGE,
           startupHealthCheckTimeoutInSeconds: 300,
           endpointName: MISTRAL_7B_INSTRUCT2_ENDPOINT_NAME,
           environment: {
+            HF_TOKEN: hfTokenSecret?.secretValue.unsafeUnwrap().toString() || "",
             SM_NUM_GPUS: JSON.stringify(1),
             MAX_INPUT_LENGTH: JSON.stringify(2048),
             MAX_TOTAL_TOKENS: JSON.stringify(4096),
@@ -205,14 +209,12 @@ export class Models extends Construct {
               (subnet) => subnet.subnetId
             ),
           },
-          container: DeepLearningContainerImage.fromDeepLearningContainerImage(
-            "huggingface-pytorch-tgi-inference",
-            "2.1.1-tgi1.3.3-gpu-py310-cu121-ubuntu20.04"
-          ),
+          container: DeepLearningContainerImage.HUGGINGFACE_PYTORCH_TGI_INFERENCE_2_1_1_TGI2_0_0_GPU_PY310_CU121_UBUNTU22_04,
           instanceType: SageMakerInstanceType.ML_G5_48XLARGE,
           startupHealthCheckTimeoutInSeconds: 300,
           endpointName: MISTRAL_8x7B_INSTRUCT2_ENDPOINT_NAME,
           environment: {
+            HF_TOKEN: hfTokenSecret?.secretValue.unsafeUnwrap().toString() || "",
             SM_NUM_GPUS: JSON.stringify(8),
             MAX_INPUT_LENGTH: JSON.stringify(24576),
             MAX_TOTAL_TOKENS: JSON.stringify(32768),

lib/shared/types.ts

@@ -88,6 +88,7 @@ export interface SystemConfig {
   };
   llms: {
     sagemaker: SupportedSageMakerModels[];
+    huggingfaceApiSecretArn?: string;
     sagemakerSchedule?: {
       enabled?: boolean;
       timezonePicker?: string;

package.json

@@ -43,7 +43,7 @@
     "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.114.1-alpha.0",
     "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.114.1-alpha.0",
     "@aws-cdk/aws-cognito-identitypool-alpha": "^2.114.1-alpha.0",
-    "@cdklabs/generative-ai-cdk-constructs": "^0.1.50",
+    "@cdklabs/generative-ai-cdk-constructs": "^0.1.122",
     "aws-cdk-lib": "2.126.0",
     "cdk-nag": "2.27.170",
     "commander": "^11.0.0",