Akki0404 commited on
Commit
5f6cf4c
·
1 Parent(s): b786564

fix grader scores to be strictly between 0 and 1

Browse files
Files changed (1) hide show
  1. environment/graders.py +7 -10
environment/graders.py CHANGED
@@ -5,29 +5,26 @@ def grade(true_label: int, action: dict, difficulty: str) -> float:
5
 
6
  if difficulty == "easy":
7
  if correct:
8
- return 1.0
9
  else:
10
- return 0.0
11
 
12
  elif difficulty == "medium":
13
  if correct:
14
- # reward confidence when correct
15
  base = 0.6
16
- bonus = 0.4 * confidence
17
  return round(base + bonus, 3)
18
  else:
19
- # penalize overconfidence when wrong
20
  penalty = 0.3 * confidence
21
- return round(max(0.0, 0.2 - penalty), 3)
22
 
23
  elif difficulty == "hard":
24
  if correct:
25
- # correct but penalize overconfidence (hard task, be humble)
26
  base = 0.5
27
- calibration_bonus = 0.5 * (1 - abs(confidence - 0.7))
28
  return round(base + calibration_bonus, 3)
29
  else:
30
  if confidence < 0.4:
31
- return 0.15 # wrong but appropriately uncertain
32
  else:
33
- return 0.0 # wrong + overconfident = worst case
 
5
 
6
  if difficulty == "easy":
7
  if correct:
8
+ return 0.95 # was 1.0
9
  else:
10
+ return 0.05 # was 0.0
11
 
12
  elif difficulty == "medium":
13
  if correct:
 
14
  base = 0.6
15
+ bonus = 0.35 * confidence # max = 0.95
16
  return round(base + bonus, 3)
17
  else:
 
18
  penalty = 0.3 * confidence
19
+ return round(max(0.05, 0.2 - penalty), 3)
20
 
21
  elif difficulty == "hard":
22
  if correct:
 
23
  base = 0.5
24
+ calibration_bonus = 0.45 * (1 - abs(confidence - 0.7))
25
  return round(base + calibration_bonus, 3)
26
  else:
27
  if confidence < 0.4:
28
+ return 0.15
29
  else:
30
+ return 0.05 # was 0.0