Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Scoring: exclude_from_ranking #3352

Merged
merged 6 commits into from
May 27, 2024
Merged

Scoring: exclude_from_ranking #3352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
32ee500
Add exclude_from_ranking for additional score extraction from metrics
chrisvanrun May 24, 2024
8b6784a
Add migration
chrisvanrun May 24, 2024
737a671
Refactor test_calculate_ranks
chrisvanrun May 26, 2024
288473d
Refactor test_calculate_ranks_with_exclusion
chrisvanrun May 26, 2024
20f13c3
Ensure rank excluded metrics are not UI decorated on leaderboards
chrisvanrun May 26, 2024
547cb57
More explicit
jmsmkn May 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions app/grandchallenge/evaluation/migrations/0053_alter_phase_extra_results_columns.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Generated by Django 4.2.11 on 2024-05-24 12:37

from django.db import migrations, models

import grandchallenge.core.validators


class Migration(migrations.Migration):

dependencies = [
("evaluation", "0052_phase_parent"),
]

operations = [
migrations.AlterField(
model_name="phase",
name="extra_results_columns",
field=models.JSONField(
blank=True,
default=list,
help_text='A JSON object that contains the extra columns from metrics.json that will be displayed on the results page. An example that will display accuracy score with error would look like this: [{"path": "accuracy.mean","order": "asc","title": "ASSD +/- std","error_path": "accuracy.std","exclude_from_ranking": true}]',
validators=[
grandchallenge.core.validators.JSONValidator(
schema={
"$schema": "http://json-schema.org/draft-06/schema#",
"definitions": {},
"items": {
"$id": "#/items",
"additionalProperties": False,
"properties": {
"error_path": {
"$id": "#/items/properties/error_path",
"default": "",
"examples": ["aggregates.dice.std"],
"pattern": "^(.*)$",
"title": "The Error Path Schema",
"type": "string",
},
"exclude_from_ranking": {
"$id": "#/items/properties/exclude_from_ranking",
"default": False,
"title": "The Exclude From Ranking Schema",
"type": "boolean",
},
"order": {
"$id": "#/items/properties/order",
"default": "",
"enum": ["asc", "desc"],
"examples": ["asc"],
"pattern": "^(asc|desc)$",
"title": "The Order Schema",
"type": "string",
},
"path": {
"$id": "#/items/properties/path",
"default": "",
"examples": ["aggregates.dice.mean"],
"pattern": "^(.*)$",
"title": "The Path Schema",
"type": "string",
},
"title": {
"$id": "#/items/properties/title",
"default": "",
"examples": ["Mean Dice"],
"pattern": "^(.*)$",
"title": "The Title Schema",
"type": "string",
},
},
"required": ["title", "path", "order"],
"title": "The Items Schema",
"type": "object",
},
"title": "The Extra Results Columns Schema",
"type": "array",
}
)
],
),
),
]
9 changes: 8 additions & 1 deletion app/grandchallenge/evaluation/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,12 @@
"examples": ["asc"],
"pattern": "^(asc|desc)$",
},
"exclude_from_ranking": {
"$id": "#/items/properties/exclude_from_ranking",
"type": "boolean",
"title": "The Exclude From Ranking Schema",
"default": False,
},
},
},
}
Expand Down Expand Up @@ -253,7 +259,8 @@ class Phase(FieldChangeMixin, HangingProtocolMixin, UUIDModel):
'"path": "accuracy.mean",'
'"order": "asc",'
'"title": "ASSD +/- std",'
'"error_path": "accuracy.std"'
'"error_path": "accuracy.std",'
'"exclude_from_ranking": true'
"}]"
),
validators=[JSONValidator(schema=EXTRA_RESULT_COLUMNS_SCHEMA)],
Expand Down
1 change: 1 addition & 0 deletions app/grandchallenge/evaluation/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,7 @@ def calculate_ranks(*, phase_pk: uuid.UUID):
*[
Metric(path=col["path"], reverse=col["order"] == phase.DESCENDING)
for col in phase.extra_results_columns
if not col.get("exclude_from_ranking", False)
],
)

Expand Down
2 changes: 1 addition & 1 deletion app/grandchallenge/evaluation/templates/evaluation/leaderboard_row.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
 ± 
{{ object.metrics_json_file|get_jsonpath:col.error_path|floatformat:object.submission.phase.score_decimal_places }}
{% endif %}
{% if object.submission.phase.scoring_method_choice != object.submission.phase.ABSOLUTE %}
{% if object.submission.phase.scoring_method_choice != object.submission.phase.ABSOLUTE and not col.exclude_from_ranking %}
 (
{{ object.rank_per_metric|get_key:col.path }}
)
Expand Down
1 change: 1 addition & 0 deletions app/grandchallenge/evaluation/views/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,6 +719,7 @@ def columns(self):
c["title"]
if self.phase.scoring_method_choice
== self.phase.ABSOLUTE
or c.get("exclude_from_ranking", False)
else f"{c['title']} (Position)"
),
sort_field="rank",
Expand Down
Loading