evalai-demonstrator-proficiency/challenge_config.yaml at challenge · EGO4D-Consortium/evalai-demonstrator-proficiency · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# Modify only the following keys in this config file. The rest have already been configured.
# - title, short_description, leaderboard_description: fill with respective benchmark details
# - leaderboard: replace with relevant benchmark metrics. See LTA challenge repo for examples.

title: "EgoExo4D Demonstrator Proficiency"  # No special characters like ":" allowed
short_description: "EgoExo4D Demonstrator Proficiency challenge"
description: templates/description.html
evaluation_details: templates/evaluation_details.html
terms_and_conditions: templates/terms_and_conditions.html
image: logo.jpg
submission_guidelines: templates/submission_guidelines.html
leaderboard_description: "POC: description of the leaderboard"
evaluation_script: evaluation_script.zip
remote_evaluation: False
is_docker_based: False
start_date: 2022-03-16 00:00:00
end_date: 2022-06-01 23:59:59
published: True

leaderboard:
  - id: 1
    schema:
      {
        "labels": ["Metric1", "Metric2", "Metric3", "Total"],
        "default_order_by": "Total",
        "metadata": {
          "Metric1": {
            "sort_ascending": True,
            "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
          },
          "Metric2": {
            "sort_ascending": True,
            "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
          }
        }
      }

challenge_phases:
  - id: 1
    name: Test Phase
    description: templates/challenge_test_phase_description.html
    leaderboard_public: True
    is_public: True
    is_submission_public: True
    start_date: 2022-03-16 00:00:00
    end_date: 2022-06-01 23:59:59
    test_annotation_file: annotations/test_annotations_testsplit.json
    codename: test
    max_submissions_per_day: 1
    max_submissions_per_month: 50
    max_submissions: 50
    default_submission_meta_attributes:
      - name: method_name
        is_visible: True
    submission_meta_attributes:
      - name: I have read and accept the T&C
        description:
        type: boolean
        required: True
    is_restricted_to_select_one_submission: False
    is_partial_submission_evaluation_enabled: False

dataset_splits:
  - id: 1
    name: Test Split
    codename: test_split

challenge_phase_splits:
  - challenge_phase_id: 1
    leaderboard_id: 1
    dataset_split_id: 1
    visibility: 1 # only visible to hosts
    leaderboard_decimal_precision: 2
    is_leaderboard_order_descending: True