-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add `examples` as embeded notebooks in the `docs/` * Remove `notebooks_folder` and `convert_notebooks` args * Update `docs/source/_toctree.yml` to include examples (WIP) * Add sample paths to examples (WIP) * Update `thumbnail.png` https://huggingface.co/datasets/huggingface/documentation-images/discussions/365 * Split partially `index.mdx` in `features.mdx` and `resources.mdx` * Add `docs/source/containers/*.mdx` (WIP) * Update `docs/source/containers/available.mdx` * Fix path to `scripts/upload_model_to_gcs.sh` * Clean `docs/source` * Add `Makefile` to auto-generate docs from examples * Add `pre_command: make docs` * (debug) Add `ls -la` before `make docs` * Fix `pre_command` to `cd` into `Google-Cloud-Containers` first * Include `examples/cloud-run` directory in `make docs` * Remove extra empty `>` lines and add `make serve` * Update `Makefile` and add `docs/sed/huggingface-tip.sed` * Add `docs/scripts/auto-generate-examples.py` * Update `Makefile` and `docs/scripts/auto-generate-examples.py` * Update "Examples" section ordering Co-authored-by: Jeff Boudier <[email protected]> * Remove emojis within `docs/source/_toctree.yml` Co-authored-by: Philipp Schmid <[email protected]> * Add `metadata` to every example under `examples` * Update `docs/scripts/auto-generate-examples.py` Remove Jupyter Markdown comment to hide the metadata after its converted from `.ipynb` to `.mdx` * Add `docs/scripts/auto-update-toctree.py` * Add `docs/source/examples` to `.gitignore` As those are automatically generated and not intended to be pushed * Update comment parsing for Jupyter Notebooks * Clean metadata from `.mdx` files (and remove if none) * Set `isExpanded: true` for top level examples * Update `docs/source/containers/available.mdx` * Fix typo in `youself`->`yourself` * Split example introduction from TL;DR * Apply suggestions from code review Co-authored-by: pagezyhf <[email protected]> * Update `containers/tgi/README.md` - Add missing `--shm-size 1g` - Fixed some wording / typos * Update and align example titles * Fix `title` for `/resources` --------- Co-authored-by: Jeff Boudier <[email protected]> Co-authored-by: Philipp Schmid <[email protected]> Co-authored-by: pagezyhf <[email protected]>
- Loading branch information
4 people
authored
Sep 21, 2024
1 parent
c7303a0
commit 241791c
Showing
32 changed files
with
601 additions
and
168 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -161,3 +161,6 @@ cython_debug/ | |
|
|
||
| # .DS_Store files | ||
| .DS_Store | ||
|
|
||
| # Auto-generated docs | ||
| docs/source/examples/ | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| .PHONY: docs clean help | ||
|
|
||
| docs: clean | ||
| @echo "Processing README.md files from examples/gke, examples/cloud-run, and examples/vertex-ai..." | ||
| @mkdir -p docs/source/examples | ||
| @echo "Converting Jupyter Notebooks to MDX..." | ||
| @doc-builder notebook-to-mdx examples/vertex-ai/notebooks/ | ||
| @echo "Auto-generating example files for documentation..." | ||
| @python docs/scripts/auto-generate-examples.py | ||
| @echo "Cleaning up generated Markdown Notebook files..." | ||
| @find examples/vertex-ai/notebooks -name "vertex-notebook.md" -type f -delete | ||
| @echo "Generating YAML tree structure and appending to _toctree.yml..." | ||
| @python docs/scripts/auto-update-toctree.py | ||
| @echo "YAML tree structure appended to docs/source/_toctree.yml" | ||
| @echo "Documentation setup complete." | ||
|
|
||
| clean: | ||
| @echo "Cleaning up generated documentation..." | ||
| @rm -rf docs/source/examples | ||
| @awk '/^# GENERATED CONTENT DO NOT EDIT!/,/^# END GENERATED CONTENT/{next} {print}' docs/source/_toctree.yml > docs/source/_toctree.yml.tmp && mv docs/source/_toctree.yml.tmp docs/source/_toctree.yml | ||
| @echo "Cleaning up generated Markdown Notebook files (if any)..." | ||
| @find examples/vertex-ai/notebooks -name "vertex-notebook.md" -type f -delete | ||
| @echo "Cleanup complete." | ||
|
|
||
| serve: | ||
| @echo "Serving documentation via doc-builder" | ||
| doc-builder preview gcloud docs/source --not_python_module | ||
|
|
||
| help: | ||
| @echo "Usage:" | ||
| @echo " make docs - Auto-generate the examples for the docs" | ||
| @echo " make clean - Remove the auto-generated docs" | ||
| @echo " make help - Display this help message" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| import os | ||
| import re | ||
|
|
||
|
|
||
| def process_readme_files(): | ||
| print("Processing README.md files from examples/gke and examples/cloud-run...") | ||
| os.makedirs("docs/source/examples", exist_ok=True) | ||
|
|
||
| for dir in ["gke", "cloud-run", "vertex-ai/notebooks"]: | ||
| for root, _, files in os.walk(f"examples/{dir}"): | ||
| for file in files: | ||
| if file == "README.md" or file == "vertex-notebook.md": | ||
| process_file(root, file, dir) | ||
|
|
||
|
|
||
| def process_file(root, file, dir): | ||
| dir_name = dir if not dir.__contains__("/") else dir.replace("/", "-") | ||
|
|
||
| file_path = os.path.join(root, file) | ||
| subdir = root.replace(f"examples/{dir}/", "") | ||
| base = os.path.basename(subdir) | ||
|
|
||
| if file_path == f"examples/{dir}/README.md": | ||
| target = f"docs/source/examples/{dir_name}-index.mdx" | ||
| else: | ||
| target = f"docs/source/examples/{dir_name}-{base}.mdx" | ||
|
|
||
| print(f"Processing {file_path} to {target}") | ||
| with open(file_path, "r") as f: | ||
| content = f.read() | ||
|
|
||
| # For Juypter Notebooks, remove the comment i.e. `<!--` and the `--!>` but keep the metadata | ||
| content = re.sub(r"<!-- (.*?) -->", r"\1", content, flags=re.DOTALL) | ||
|
|
||
| # Replace image and link paths | ||
| content = re.sub( | ||
| r"\(\./(imgs|assets)/([^)]*\.png)\)", | ||
| r"(https://raw.githubusercontent.com/huggingface/Google-Cloud-Containers/main/" | ||
| + root | ||
| + r"/\1/\2)", | ||
| content, | ||
| ) | ||
| content = re.sub( | ||
| r"\(\.\./([^)]+)\)", | ||
| r"(https://github.com/huggingface/Google-Cloud-Containers/tree/main/examples/" | ||
| + dir | ||
| + r"/\1)", | ||
| content, | ||
| ) | ||
| content = re.sub( | ||
| r"\(\.\/([^)]+)\)", | ||
| r"(https://github.com/huggingface/Google-Cloud-Containers/tree/main/" | ||
| + root | ||
| + r"/\1)", | ||
| content, | ||
| ) | ||
|
|
||
| # Regular expression to match the specified blocks | ||
| pattern = r"> \[!(NOTE|WARNING)\]\n((?:> .*\n)+)" | ||
|
|
||
| def replacement(match): | ||
| block_type = match.group(1) | ||
| content = match.group(2) | ||
|
|
||
| # Remove '> ' from the beginning of each line and strip whitespace | ||
| lines = [ | ||
| line.lstrip("> ").strip() for line in content.split("\n") if line.strip() | ||
| ] | ||
|
|
||
| # Determine the Tip type | ||
| tip_type = " warning" if block_type == "WARNING" else "" | ||
|
|
||
| # Construct the new block | ||
| new_block = f"<Tip{tip_type}>\n\n" | ||
| new_block += "\n".join(lines) | ||
| new_block += "\n\n</Tip>\n" | ||
|
|
||
| return new_block | ||
|
|
||
| # Perform the transformation | ||
| content = re.sub(pattern, replacement, content, flags=re.MULTILINE) | ||
|
|
||
| # Remove blockquotes | ||
| content = re.sub(r"^(>[ ]*)+", "", content, flags=re.MULTILINE) | ||
|
|
||
| # Check for remaining relative paths | ||
| if re.search(r"\(\.\./|\(\./", content): | ||
| print("WARNING: Relative paths still exist in the processed file.") | ||
| print( | ||
| "The following lines contain relative paths, consider replacing those with GitHub URLs instead:" | ||
| ) | ||
| for i, line in enumerate(content.split("\n"), 1): | ||
| if re.search(r"\(\.\./|\(\./", line): | ||
| print(f"{i}: {line}") | ||
| else: | ||
| print("No relative paths found in the processed file.") | ||
|
|
||
| with open(target, "w") as f: | ||
| f.write(content) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| process_readme_files() |
Oops, something went wrong.