diff --git a/ai-ml/trainium-inferentia/examples/gradio-ui/gradio-app.py b/ai-ml/trainium-inferentia/examples/gradio-ui/gradio-app.py index d257d0017..7b8f6f535 100644 --- a/ai-ml/trainium-inferentia/examples/gradio-ui/gradio-app.py +++ b/ai-ml/trainium-inferentia/examples/gradio-ui/gradio-app.py @@ -21,15 +21,11 @@ def text_generation(message, history): full_output = response.text # Removing the original question from the output - answer_only = full_output.replace(prompt, "", 1).strip() - - # Remove any leading/trailing square brackets and double quotes - answer_only = answer_only.strip('["]') + answer_only = full_output.replace(prompt, "", 1).strip('["]?\n') # Safety filter to remove harmful or inappropriate content answer_only = filter_harmful_content(answer_only) - - return f"

{answer_only}

" # Return text with preserved newlines + return answer_only except requests.exceptions.RequestException as e: # Handle any request exceptions (e.g., connection errors) return f"AI: Error: {str(e)}" @@ -46,7 +42,7 @@ def filter_harmful_content(text): # Define the Gradio ChatInterface chat_interface = gr.ChatInterface( text_generation, - chatbot=gr.Chatbot(height=400, line_breaks=True), + chatbot=gr.Chatbot(line_breaks=True), textbox=gr.Textbox(placeholder="Ask me a question", container=False, scale=7), title="Llama2 AI Chat", description="Ask me any question", diff --git a/ai-ml/trainium-inferentia/examples/ray-serve/llama2-inf2/ray-service-llama2.yaml b/ai-ml/trainium-inferentia/examples/ray-serve/llama2-inf2/ray-service-llama2.yaml index be2026c3b..9fc6926e9 100644 --- a/ai-ml/trainium-inferentia/examples/ray-serve/llama2-inf2/ray-service-llama2.yaml +++ b/ai-ml/trainium-inferentia/examples/ray-serve/llama2-inf2/ray-service-llama2.yaml @@ -108,7 +108,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: gradio-ingress + name: llama2-ingress namespace: llama2 annotations: nginx.ingress.kubernetes.io/rewrite-target: "/$1" diff --git a/website/docs/gen-ai/index.md b/website/docs/gen-ai/index.md new file mode 100644 index 000000000..fe479145b --- /dev/null +++ b/website/docs/gen-ai/index.md @@ -0,0 +1,19 @@ +--- +sidebar_position: 1 +sidebar_label: Overview +--- + +# Gen AI on EKS + +Welcome to Gen AI on Amazon Elastic Kubernetes Service (EKS), your gateway to harnessing the power of Large Language Models (LLMs) for a wide range of applications. This introduction page serves as your starting point to explore our offerings for Training, Fine-tuning, and Inference using various LLMs, including BERT-Large, Llama2, Stable Diffusion, and more. + +## [Training](https://awslabs.github.io/docs/category/training-on-eks) +Are you ready to dive into the world of LLMs and train models for your specific needs? Discover our comprehensive Training resources to get started. + +## [Fine-tuning](https://awslabs.github.io/docs/category/training-on-eks) +Fine-tuning LLMs is crucial for tailoring them to your specific tasks. Explore our Fine-tuning section to learn how to adapt LLMs to your unique requirements. + +## [Inference](https://awslabs.github.io/docs/category/inference-on-eks) +Unlock the potential of LLMs for powerful inference tasks. Our Inference resources will guide you through deploying LLMs effectively. + +Whether you're an experienced practitioner or new to the field, our Gen AI on EKS capabilities empower you to harness the latest advancements in language modeling. Dive into each section to begin your journey. diff --git a/website/docs/gen-ai/inference/Llama2.md b/website/docs/gen-ai/inference/Llama2.md new file mode 100644 index 000000000..a5683c63b --- /dev/null +++ b/website/docs/gen-ai/inference/Llama2.md @@ -0,0 +1,310 @@ +--- +title: Llama-2 on EKS +sidebar_position: 1 +--- +import CollapsibleContent from '../../../src/components/CollapsibleContent'; + + +:::danger + +Note: Use of this Llama-2 model is governed by the Meta license. +In order to download the model weights and tokenizer, please visit the [website](https://ai.meta.com/) and accept our License before requesting access here. + +::: + +:::info + +We are actively enhancing this blueprint to incorporate improvements in observability, logging, and scalability aspects. + +::: + + +# Llama-2-Chat on EKS: Deploying Llama-2-13b Chat Model with Ray Serve and Gradio +Welcome to the comprehensive guide on deploying the [Meta Llama-2-13b chat](https://ai.meta.com/llama/#inside-the-model) model on Amazon Elastic Kubernetes Service (EKS) using [Ray Serve](https://docs.ray.io/en/latest/serve/index.html). +In this tutorial, you will not only learn how to harness the power of Llama-2, but also gain insights into the intricacies of deploying large language models (LLMs) efficiently, particularly on [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/) instances, such as `inf2.24xlarge` and `inf2.48xlarge`, which are optimized for deploying and scaling large language models. + +### What is Llama-2? +Llama-2 is a pretrained large language model (LLM) trained on 2 trillion tokens of text and code. It is one of the largest and most powerful LLMs available today. Llama-2 can be used for a variety of tasks, including natural language processing, text generation, and translation. + +#### Llama-2-chat +Llama-2 is a remarkable language model that has undergone a rigorous training process. It starts with pretraining using publicly available online data. An initial version of Llama-2-chat is then created through supervised fine-tuning. +Following that, `Llama-2-chat` undergoes iterative refinement using Reinforcement Learning from Human Feedback (`RLHF`), which includes techniques like rejection sampling and proximal policy optimization (`PPO`). +This process results in a highly capable and fine-tuned language model that we will guide you to deploy and utilize effectively on **Amazon EKS** with **Ray Serve**. + +Llama-2 is available in three different model sizes: + +- **Llama-2-70b:** This is the largest Llama-2 model, with 70 billion parameters. It is the most powerful Llama-2 model and can be used for the most demanding tasks. +- **Llama-2-13b:** This is a medium-sized Llama-2 model, with 13 billion parameters. It is a good balance between performance and efficiency, and can be used for a variety of tasks. +- **Llama-2-7b:** This is the smallest Llama-2 model, with 7 billion parameters. It is the most efficient Llama-2 model and can be used for tasks that do not require the highest level of performance. + +### **Which Llama-2 model size should I use?** +The best Llama-2 model size for you will depend on your specific needs. and it may not always be the largest model for achieving the highest performance. It's advisable to evaluate your needs and consider factors such as computational resources, response time, and cost-efficiency when selecting the appropriate Llama-2 model size. The decision should be based on a comprehensive assessment of your application's goals and constraints. + +## Inference on AWS Neuron Instances: Unlocking the Full Potential of Llama-2 +**Llama-2** can be deployed on a variety of hardware platforms, each with its own set of advantages. However, when it comes to maximizing the efficiency, scalability, and cost-effectiveness of Llama-2, [AWS Neuron instances](https://aws.amazon.com/ec2/instance-types/inf2/) shine as the optimal choice. + +**Scalability and Availability** +One of the key challenges in deploying large language models (`LLMs`) like Llama-2 is the scalability and availability of suitable hardware. Traditional `GPU` instances often face scarcity due to high demand, making it challenging to provision and scale resources effectively. +In contrast, AWS Neuron instances, such as `trn1.32xlarge`, `trn1n.32xlarge`, `inf2.24xlarge` and `inf2.48xlarge`, are tailor-made for LLM workloads. They offer both scalability and availability, ensuring that you can deploy and scale your `Llama-2` models as needed, without resource bottlenecks or delays. + +**Cost Optimization:** +Running LLMs on traditional GPU instances can be cost-prohibitive, especially given the scarcity of GPUs and their competitive pricing. +AWS Neuron instances provide a cost-effective alternative. By offering dedicated hardware optimized for AI and machine learning tasks, Neuron instances allow you to achieve top-notch performance at a fraction of the cost. +This cost optimization enables you to allocate your budget efficiently, making LLM deployment accessible and sustainable. + +**Performance Boost** +While Llama-2 can achieve high-performance inference on GPUs, Neuron accelerators take performance to the next level. Neuron accelerators are purpose-built for machine learning workloads, providing hardware acceleration that significantly enhances Llama-2's inference speeds. This translates to faster response times and improved user experiences when deploying Llama-2 on Neuron instances. + +### Model Specification +The table provides information about the different sizes of Llama-2 models, their weights, and the hardware requirements for deploying them. This information can be used to design the infrastructure required to deploy any size of Llama-2 model. For example, if you want to deploy the `Llama-2-13b-chat` model, you will need to use an instance type with at least `26 GB` of total accelerator memory. + +| Model | Weights | Bytes | Parameter Size (Billions) | Total Accelerator Memory (GB) | Accelerator Memory Size for NeuronCore (GB) | Required Neuron Cores | Required Neuron Accelerators | Instance Type | tp_degree | +|-----------------|---------|-------|-----------------------------|------------------------------|---------------------------------------------|-----------------------|-----------------------------|-----------------|-----------| +| Meta/Llama-2-70b | float16 | 2 | 70 | 140 | 16 | 9 | 5 | inf2.48x | 24 | +| Meta/Llama-2-13b | float16 | 2 | 13 | 26 | 16 | 2 | 1 | inf2.24x | 12 | +| Meta/Llama-2-7b | float16 | 2 | 7 | 14 | 16 | 1 | 1 | inf2.24x | 12 | + +### Example usecase +A company wants to deploy a Llama-2 chatbot to provide customer support. The company has a large customer base and expects to receive a high volume of chat requests at peak times. The company needs to design an infrastructure that can handle the high volume of requests and provide a fast response time. + +The company can use Inferentia2 instances to scale its Llama-2 chatbot efficiently. Inferentia2 instances are specialized hardware accelerators for machine learning tasks. They can provide up to 20x better performance and up to 7x lower cost than GPUs for machine learning workloads. + +The company can also use Ray Serve to horizontally scale its Llama-2 chatbot. Ray Serve is a distributed framework for serving machine learning models. It can automatically scale your models up or down based on demand. + +To scale its Llama-2 chatbot, the company can deploy multiple Inferentia2 instances and use Ray Serve to distribute the traffic across the instances. This will allow the company to handle a high volume of requests and provide a fast response time. + +## Solution Architecture +In this section, we will delve into the architecture of our solution, which combines Llama-2 model, [Ray Serve](https://docs.ray.io/en/latest/serve/index.html) and [Inferentia2](https://aws.amazon.com/ec2/instance-types/inf2/) on Amazon EKS. + +:::info + +COMING SOON + +::: + + +## Deploying the Solution +To get started with deploying `Llama-2-13b chat` on [Amazon EKS](https://aws.amazon.com/eks/), we will cover the necessary prerequisites and guide you through the deployment process step by step. +This includes setting up the infrastructure, deploying the **Ray cluster**, and creating the [Gradio](https://www.gradio.app/) WebUI app. + +Prerequisites}> +Before we begin, ensure you have all the prerequisites in place to make the deployment process smooth and hassle-free. +nsure that you have installed the following tools on your machine. + +1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) + +### Deploy + +Clone the repository + +```bash +git clone https://github.com/awslabs/data-on-eks.git +``` + +Navigate into one of the example directories and run `install.sh` script + +**Important Note:** Ensure that you update the region in the `variables.tf` file before deploying the blueprint. +Additionally, confirm that your local region setting matches the specified region to prevent any discrepancies. +For example, set your `export AWS_DEFAULT_REGION=""` to the desired region: + +```bash +cd data-on-eks/ai-ml/trainium/ && chmod +x install.sh +./install.sh +``` + +### Verify the resources + +Verify the Amazon EKS Cluster + +```bash +aws eks describe-cluster --name trainium +``` + +```bash +# Creates k8s config file to authenticate with EKS +aws eks --region us-west-2 update-kubeconfig --name trainium + +kubectl get nodes # Output shows the EKS Managed Node group nodes +``` + + + +## Deploying the Ray Cluster with Llama-2-Chat Model +Once the `Trainium on EKS` Cluster is deployed, you can proceed to use `kubectl` to deploy the `ray-service-Llama-2.yaml`. + +In this step, we will deploy the Ray Serve cluster, which comprises one `Head Pod` on `x86 CPU` instances using Karpenter autoscaling, as well as `Ray workers` on `Inf2.48xlarge` instances, autoscaled by [Karpenter](https://karpenter.sh/). + +Let's take a closer look at the key files used in this deployment and understand their functionalities before proceeding with the deployment: + +- **ray_serve_Llama-2.py:** +This script uses FastAPI, Ray Serve, and PyTorch-based Hugging Face Transformers to create an efficient API for text generation using the [NousResearch/Llama-2-13b-chat-hf](https://huggingface.co/NousResearch/Llama-2-13b-chat-hf) language model. +Alternatively, users have the flexibility to switch to the [meta-llama/Llama-2-13b-chat-hf](https://huggingface.co/meta-llama/Llama-2-13b-chat-hf) model. The script establishes an endpoint that accepts input sentences and efficiently generates text outputs, benefiting from Neuron acceleration for enhanced performance. With its high configurability, users can fine-tune model parameters to suit a wide range of natural language processing applications, including chatbots and text generation tasks. + +- **ray-service-Llama-2.yaml:** +This Ray Serve YAML file serves as a Kubernetes configuration for deploying the Ray Serve service, facilitating efficient text generation using the `Llama-2-13b-chat` model. +It defines a Kubernetes namespace named `Llama-2` to isolate resources. Within the configuration, the `RayService` specification, named `Llama-2-service`, is created and hosted within the `Llama-2` namespace. The `RayService` specification leverages the Python script `ray_serve_Llama-2.py` (copied into the Dockerfile located within the same folder) to create the Ray Serve service. +The Docker image used in this example is publicly available on Amazon Elastic Container Registry (ECR) for ease of deployment. +Users can also modify the Dockerfile to suit their specific requirements and push it to their own ECR repository, referencing it in the YAML file. + +### Deploy the Llama-2-Chat Model + +**Ensure the cluster is configured locally** +```bash +aws eks --region us-west-2 update-kubeconfig --name trainium +``` + +**Deploy RayServe Cluster** + +```bash +cd ai-ml/trainium-inferentia/examples/ray-serve/Llama-2-inf2 +kubectl apply -f ray-service-Llama-2.yaml +``` + +Verify the deployment by running the following commands + +:::info + +The deployment process may take up to 10 minutes. The Head Pod is expected to be ready within 2 to 3 minutes, while the Ray Serve worker pod may take up to 10 minutes for image retrieval and Model deployment from Huggingface. + +::: + +```text +$ kubectl get all -n Llama-2 + +NAME READY STATUS RESTARTS AGE +pod/Llama-2-service-raycluster-bt7bs-head-nhdct 0/1 ContainerCreating 0 68s +pod/service-raycluster-bt7bs-worker-inf2-worker-group-wtv47 0/1 Pending 0 68s + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/Llama-2-service NodePort 172.20.123.199 6379:31306/TCP,8265:30765/TCP,10001:32101/TCP,8000:30807/TCP,52365:31237/TCP,8080:31221/TCP 69s + +$ kubectl get ingress -n Llama-2 + +NAME CLASS HOSTS ADDRESS PORTS AGE +Llama-2-ingress nginx * k8s-ingressn-ingressn-randomid-randomid.elb.us-west-2.amazonaws.com 80 2m4s + +``` + +Now, you can access the Ray Dashboard from the Load balancer URL below. + + http:///dashboard/#/serve + +If you don't have access to a public Load Balancer, you can use port-forwarding and browse the Ray Dashboard using localhost with the following command: + +```bash +kubectl port-forward svc/Llama-2-service 8265:8265 -n Llama-2 + +# Open the link in the browser +http://localhost:8265/ + +``` + +From this webpage, you will be able to monitor the progress of Model deployment, as shown in the image below: + +![Ray Dashboard](img/ray-dashboard.png) + +### To Test the Llama-2-Chat Model +Once you see the status of the model deployment is in `running` state then you can start using Llama-2-chat. + +You can use the following URL with a query added at the end of the URL. + + http:///serve/infer?sentence=what is data parallelism and tensor parallelisma and the diffrences + +You will see an output like this in your browser: + +![Chat Output](img/llama-2-chat-ouput.png) + +## Deploying the Gradio WebUI App +Discover how to create a user-friendly chat interface using [Gradio](https://www.gradio.app/) that integrates seamlessly with deployed models. + +Let's deploy Gradio app locally on your machine to interact with the LLama-2-Chat model deployed using RayServe. + +:::info + +The Gradio app interacts with the locally exposed service created solely for the demonstration. Alternatively, you can deploy the Gradio app on EKS as a Pod with Ingress and Load Balancer for wider accessibility. + +::: + +### Execute Port Forward to the Llama-2 Ray Service +First, execute a port forward to the Llama-2 Ray Service using kubectl: + +```bash +kubectl port-forward svc/Llama-2-service 8000:8000 -n Llama-2 +``` + +### Deploy Gradio WebUI Locally + +#### Create a Virtual Environment +Create a Python virtual environment in your machine for the Gradio application: + +```bash +cd ai-ml/trainium-inferentia/examples/gradio-ui +python3 -m venv .venv +source .venv/bin/activate +``` + +#### Install Gradio ChatBot app +Install all the Gradio WebUI app dependencies with pip + +```bash +pip install gradio requests +``` + +#### Invoke the WebUI +Run the Gradio WebUI using the following command: + +NOTE: `gradio-app.py` refers to the port forward url. e.g., `service_name = "http://localhost:8000" ` + +```bash +python gradio-app.py +``` + +You should see output similar to the following: + +```text +Using cache from ~/data-on-eks/ai-ml/trainium-inferentia/examples/gradio-ui/gradio_cached_examples/16' directory. If method or examples have changed since last caching, delete this folder to clear cache. + +Running on local URL: http://127.0.0.1:7860 + +To create a public link, set `share=True` in `launch()`. +``` + +#### 2.4. Access the WebUI from Your Browser +Open your web browser and access the Gradio WebUI by navigating to the following URL: + +http://127.0.0.1:7860 + +You should now be able to interact with the Gradio application from your local machine. + +![Gradio Llama-2 AI Chat](img/gradio-llama-ai-chat.png) + +## Conclusion +In conclusion, you will have successfully deployed the **Llama-2-13b chat** model on EKS with Ray Serve and created a chatGPT-style chat web UI using Gradio. +This opens up exciting possibilities for natural language processing and chatbot development. + +In summary, when it comes to deploying and scaling Llama-2, AWS Neuron instances offer a compelling advantage. +They provide the scalability, cost optimization, and performance boost needed to make running large language models efficient and accessible, all while overcoming the challenges associated with the scarcity of GPUs. +Whether you're building chatbots, natural language processing applications, or any other LLM-driven solution, Neuron instances empower you to harness the full potential of Llama-2 on the AWS cloud. + +## Cleanup +Finally, we'll provide instructions for cleaning up and deprovisioning the resources when they are no longer needed. + +**Step1:** Cancel the execution of the `python gradio-app.py` + +**Step2:** Delete Ray Cluster + +```bash +cd ai-ml/trainium-inferentia/examples/ray-serve/Llama-2-inf2 +kubectl delete -f ray-service-Llama-2.yaml +``` + +**Step3:** Cleanup the EKS Cluster +This script will cleanup the environment using `-target` option to ensure all the resources are deleted in correct order. + +```bash +export AWS_DEAFULT_REGION="DEPLOYED_EKS_CLUSTER_REGION>" +cd data-on-eks/ai-ml/trainium/ && chmod +x cleanup.sh +./cleanup.sh +``` diff --git a/website/docs/gen-ai/inference/StableDiffusion.md b/website/docs/gen-ai/inference/StableDiffusion.md new file mode 100644 index 000000000..2d5210cb8 --- /dev/null +++ b/website/docs/gen-ai/inference/StableDiffusion.md @@ -0,0 +1,12 @@ +--- +title: Stable Diffusion on EKS +sidebar_position: 2 +--- + +:::info + +COMING SOON + +Please note that this section is currently a work in progress and will serve as a comprehensive collection of resources for running data and ML workloads on EKS. + +::: diff --git a/website/docs/gen-ai/inference/_category_.json b/website/docs/gen-ai/inference/_category_.json new file mode 100644 index 000000000..7c1602672 --- /dev/null +++ b/website/docs/gen-ai/inference/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Inference on EKS", + "position": 1, + "link": { + "type": "generated-index" + } +} diff --git a/website/docs/gen-ai/inference/img/gradio-llama-ai-chat.png b/website/docs/gen-ai/inference/img/gradio-llama-ai-chat.png new file mode 100644 index 000000000..4e667cf2a Binary files /dev/null and b/website/docs/gen-ai/inference/img/gradio-llama-ai-chat.png differ diff --git a/website/docs/gen-ai/inference/img/llama-2-chat-ouput.png b/website/docs/gen-ai/inference/img/llama-2-chat-ouput.png new file mode 100644 index 000000000..b98458b69 Binary files /dev/null and b/website/docs/gen-ai/inference/img/llama-2-chat-ouput.png differ diff --git a/website/docs/gen-ai/inference/img/ray-dashboard.png b/website/docs/gen-ai/inference/img/ray-dashboard.png new file mode 100644 index 000000000..b47bf7b1c Binary files /dev/null and b/website/docs/gen-ai/inference/img/ray-dashboard.png differ diff --git a/website/docs/gen-ai/training/BERT-Large.md b/website/docs/gen-ai/training/BERT-Large.md new file mode 100644 index 000000000..0fa1ad660 --- /dev/null +++ b/website/docs/gen-ai/training/BERT-Large.md @@ -0,0 +1,12 @@ +--- +title: BERT-Large on EKS +sidebar_position: 1 +--- + +:::info + +COMING SOON + +Please note that this section is currently a work in progress and will serve as a comprehensive collection of resources for running data and ML workloads on EKS. + +::: diff --git a/website/docs/gen-ai/training/_category_.json b/website/docs/gen-ai/training/_category_.json new file mode 100644 index 000000000..38d3dbcb7 --- /dev/null +++ b/website/docs/gen-ai/training/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Training on EKS", + "position": 2, + "link": { + "type": "generated-index" + } +} diff --git a/website/docs/workshop/intro.md b/website/docs/workshop/intro.md deleted file mode 100644 index 4ada5c471..000000000 --- a/website/docs/workshop/intro.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -sidebar_position: 1 -sidebar_label: Introduction ---- - -# Introduction - -:::info - -COMING SOON - -Please note that this section is currently a work in progress and will serve as a comprehensive workshop for hands-on learning with data and ML workloads on EKS. -It will include step-by-step instructions, code samples, exercises, and other resources to help you gain practical experience and deepen your understanding of running data and ML workloads on EKS. - -::: diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index bec1cf8c6..c65b4d0d2 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -61,6 +61,12 @@ const config = { position: 'left', label: 'Introduction', }, + { + type: 'doc', + docId: 'gen-ai/index', + position: 'left', + label: 'Gen AI' + }, { type: 'doc', docId: 'blueprints/amazon-emr-on-eks/index', @@ -79,12 +85,6 @@ const config = { position: 'left', label: 'Benchmarks' }, - { - type: 'doc', - docId: 'workshop/intro', - position: 'left', - label: 'Workshop' - }, { type: 'doc', docId: 'resources/intro', diff --git a/website/sidebars.js b/website/sidebars.js index 7dc8c0bd4..e69016bab 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -15,12 +15,11 @@ const sidebars = { // By default, Docusaurus generates a sidebar from the docs folder structure // docs: [{type: 'autogenerated', dirName: '.'}], - // But you can create a sidebar manually + genai: [{type: 'autogenerated', dirName: 'gen-ai'}], blueprints: [{type: 'autogenerated', dirName: 'blueprints'}], bestpractices: [{type: 'autogenerated', dirName: 'bestpractices'}], benchmarks: [{type: 'autogenerated', dirName: 'benchmarks'}], - workshop: [{type: 'autogenerated', dirName: 'workshop'}], resources: [{type: 'autogenerated', dirName: 'resources'}], };