diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml new file mode 100644 index 0000000..0f903d8 --- /dev/null +++ b/.github/workflows/build-docs.yaml @@ -0,0 +1,43 @@ +name: build-docs + +on: + push: + branches: + - main + - "build-docs-*" + pull_request: + branches: + - main +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + - run: uv pip install --python ${Python_ROOT_DIR} '../../requirements.txt' + - name: Build Documentation + if: github.ref != 'refs/heads/main' + run: mkdocs build + - name: Build + Deploy Documentation + if: github.ref == 'refs/heads/main' + run: mkdocs gh-deploy --force diff --git a/.github/workflows/check-links.yaml b/.github/workflows/check-links.yaml new file mode 100644 index 0000000..9588b70 --- /dev/null +++ b/.github/workflows/check-links.yaml @@ -0,0 +1,15 @@ +name: Check Markdown links + +on: + workflow_dispatch: + push: + pull_request: + schedule: + - cron: "0 0 1 * *" + +jobs: + markdown-link-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: gaurav-nelson/github-action-markdown-link-check@v1 diff --git a/docs/_footer.md b/docs/_footer.md new file mode 100644 index 0000000..5ba70f9 --- /dev/null +++ b/docs/_footer.md @@ -0,0 +1,9 @@ +
+ +- :material-bug:{ .lg .middle } __Something broken?__ + [:octicons-arrow-right-24: Report bug](https://github.com/princeton-nlp/SWE-agent/issues/new?template=bug_report.yml) + +- :material-help:{ .lg .middle } __Something unclear?__ + [:octicons-arrow-right-24: Ask question](https://github.com/princeton-nlp/SWE-agent/issues/new?template=question.yml) + +
diff --git a/docs/docs/faq.md b/docs/docs/faq.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/docs/usage.md b/docs/docs/usage.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..940ca25 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,41 @@ +# Scicode-bench: How good are LMs at science? + +Scicode-bench uses more than XX handcollected problem sets from various disciplines to evaluate the ability of LMs to solve any scientific challenge. + +Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula. + +
+ +- :material-book:{ .lg .middle } __Leaderboard__ + + --- + + How good are LMs at science, really? + + [:octicons-arrow-right-24: Browse the results](leaderboard.md) + +- :material-book:{ .lg .middle } __Preprint__ + + --- + + Learn all the details + + [:octicons-arrow-right-24: Read the preprint](https://arxiv.com) +
+ +commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. + +Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc. + +
+ + +- :material-play:{ .lg .middle } __Installation & usage__ + + --- + + Learn how to evaluate your model + + [:octicons-arrow-right-24: Read the docs](docs/index.md) + +
diff --git a/docs/leaderboard.md b/docs/leaderboard.md new file mode 100644 index 0000000..19aae9e --- /dev/null +++ b/docs/leaderboard.md @@ -0,0 +1,14 @@ +# Leaderboard + +
+ +| Date | Authors | Model | Score | Notes | +| -------- | ------- | --- | --- | --- | +| January | 0.8 | +| February | 0.7 | +| March | 0.6 | + +
+ +!!! tip "How to submit" + Want to submit your own model? Head over to the [documentation](docs/index.md). \ No newline at end of file diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 0000000..8e70e41 --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,6 @@ +{% extends "base.html" %} + +{% block announce %} + +Some news here: We've just open-sourced our repository. +{% endblock %} diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..dc81299 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,65 @@ +site_name: SciCode Benchmark +theme: + name: material + custom_dir: docs/overrides + icon: + repo: fontawesome/brands/github + # logo: assets/swe-agent.svg + palette: + - media: "(prefers-color-scheme)" + toggle: + icon: material/brightness-auto + name: Switch to light mode + - scheme: default + primary: black + accent: deep orange + media: "(prefers-color-scheme: light)" + toggle: + icon: material/weather-night + name: Switch to dark mode + - scheme: slate + primary: black + accent: deep orange + media: "(prefers-color-scheme: dark)" + toggle: + icon: material/weather-sunny + name: Switch to light mode + features: + - navigation.tabs + - navigation.tabs.sticky + - navigation.indexes + - content.action.edit + - navigation.footer + - content.code.copy + - content.footnote.tooltips + - header.autohide + - announce.dismiss +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.magiclink + - footnotes + - attr_list + - md_in_html + - pymdownx.snippets: + check_paths: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg +nav: + - Overview: index.md + - Leaderboard: leaderboard.md + - Preprint: "https://arxiv.org" + - Documentation: + - "docs/index.md" + - Quickstart: docs/quickstart.md + - Usage: docs/usage.md + - FAQ: docs/faq.md +plugins: + - glightbox + - search + - include-markdown +repo_url: https://github.com/pscicode-bench/website-draft +repo_name: sicode-bench/website-draft +edit_uri: edit/main/docs/ diff --git a/mlc_config.json b/mlc_config.json new file mode 100644 index 0000000..02d3b90 --- /dev/null +++ b/mlc_config.json @@ -0,0 +1,13 @@ +{ + "ignorePatterns": [ + { + "pattern": "https://github.com/issues?.*" + }, + { + "pattern": "https://github.com/?.*/pull/?.*" + }, + { + "pattern": ".*localhost.*" + } + ] +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..81c65d7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +mkdocs +mkdocs-material +mkdocs-glightbox +mkdocs-include-markdown-plugin