Merge branch 'run-llama:main' into main

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: false
 contact_links:
   - name: LlamaIndex Documentation
-    url: https://gpt-index.readthedocs.io/en/latest/index.html
+    url: https://docs.llamaindex.ai/en/latest/index.html
     about: Please review/search the latest documentation before posting an issue.
   - name: LlamaIndex Community Support
     url: https://discord.gg/dGcwcsnxhU

.github/ISSUE_TEMPLATE/question-form.yml

@@ -12,7 +12,7 @@ body:
     id: pre-question
     attributes:
       label: Question Validation
-      description: By submitting this issue, you agree that you've previously search our [documentation](https://gpt-index.readthedocs.io/en/latest/index.html) and [discord](https://discord.gg/dGcwcsnxhU) and did not find a proper answer to your question.
+      description: By submitting this issue, you agree that you've previously search our [documentation](https://docs.llamaindex.ai/en/latest/index.html) and [discord](https://discord.gg/dGcwcsnxhU) and did not find a proper answer to your question.
       options:
         - label: I have searched both the documentation and discord for an answer.
           required: true

.github/workflows/lint.yml

@@ -29,9 +29,9 @@ jobs:
         uses: snok/install-poetry@v1
         with:
           version: ${{ env.POETRY_VERSION }}
-      - name: Install deps
+      - name: Install pre-commit
         shell: bash
-        run: poetry install --with dev
+        run: poetry run pip install pre-commit
       - name: Run linter
         shell: bash
         run: poetry run make lint

.github/workflows/publish_release.yml

@@ -7,13 +7,34 @@ on:
 
   workflow_dispatch:
 
+env:
+  POETRY_VERSION: "1.6.1"
+  PYTHON_VERSION: "3.9"
+
 jobs:
   build-n-publish:
     name: Build and publish to PyPI
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
+      - name: Set up python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+      - name: Install deps
+        shell: bash
+        run: pip install -e .
+      - name: Cache tiktoken and nltk files
+        shell: bash
+        run: python -c "from llama_index import get_tokenizer; get_tokenizer()"
+      - name: Clean up zip files
+        shell: bash
+        run: rm -rf llama_index/_static/nltk_cache/corpora/stopwords.zip llama_index/_static/nltk_cache/tokenizers/punkt.zip
       - name: Build and publish to pypi
         uses: JRubics/[email protected]
         with:

.gitignore

@@ -1,3 +1,4 @@
+llama_index/_static
 .DS_Store
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -141,9 +142,15 @@ dmypy.json
 modules/
 *.swp
 
+# VsCode
+.vscode
+
 # pipenv
 Pipfile
 Pipfile.lock
 
 # pyright
 pyrightconfig.json
+
+# persist dir for chromadb test
+/data/

.pre-commit-config.yaml

@@ -10,24 +10,44 @@ repos:
       - id: check-merge-conflict
       - id: check-symlinks
       - id: check-toml
+        exclude: llama_index/_static
       - id: check-yaml
+        exclude: llama_index/_static
       - id: detect-private-key
       - id: end-of-file-fixer
+        exclude: llama_index/_static
       - id: mixed-line-ending
+        exclude: llama_index/_static
       - id: trailing-whitespace
+        exclude: llama_index/_static
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.1.2
+    rev: v0.1.5
 
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+        exclude: llama_index/_static
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 23.10.1
     hooks:
       - id: black-jupyter
         name: black-src
         alias: black
-        exclude: docs/
+        exclude: ^(docs/|llama_index/_static)
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.0.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          [
+            "types-requests",
+            "types-Deprecated",
+            "types-redis",
+            "types-setuptools",
+            "types-PyYAML",
+            "types-protobuf==4.24.0.4",
+          ]
+        exclude: ^(docs/|llama_index/_static)
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 23.10.1
     hooks:
@@ -51,11 +71,14 @@ repos:
     rev: v3.0.3
     hooks:
       - id: prettier
+        exclude: llama_index/_static
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:
       - id: codespell
         additional_dependencies: [tomli]
+        exclude: llama_index/_static
+        args: ["--ignore-words-list", "nin"]
   - repo: https://github.com/srstevenson/nb-clean
     rev: 3.1.0
     hooks:
@@ -65,4 +88,4 @@ repos:
     rev: v0.23.1
     hooks:
       - id: toml-sort-fix
-        exclude: poetry.lock
+        exclude: ^(poetry.lock|llama_index/_static)

.readthedocs.yaml

@@ -9,6 +9,13 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
+  commands:
+    - python -mvirtualenv $READTHEDOCS_VIRTUALENV_PATH
+    - python -m pip install --upgrade --no-cache-dir pip setuptools
+    - python -m pip install --upgrade --no-cache-dir sphinx readthedocs-sphinx-ext
+    - python -m pip install --exists-action=w --no-cache-dir -r docs/requirements.txt
+    - cat docs/conf.py
+    - python -m sphinx -T -E -b html -d ./docs/_build/doctrees -D language=en ./docs $READTHEDOCS_OUTPUT/html -j 8
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:

.vscode/settings.json

@@ -2,7 +2,7 @@
   "[python]": {
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
-      "source.fixAll": true
+      "source.fixAll": "explicit"
     },
     "editor.defaultFormatter": "ms-python.black-formatter"
   },