fix grader scores to be strictly between 0 and 1
Browse files- environment/graders.py +7 -10
environment/graders.py
CHANGED
|
@@ -5,29 +5,26 @@ def grade(true_label: int, action: dict, difficulty: str) -> float:
|
|
| 5 |
|
| 6 |
if difficulty == "easy":
|
| 7 |
if correct:
|
| 8 |
-
return 1.0
|
| 9 |
else:
|
| 10 |
-
return 0.0
|
| 11 |
|
| 12 |
elif difficulty == "medium":
|
| 13 |
if correct:
|
| 14 |
-
# reward confidence when correct
|
| 15 |
base = 0.6
|
| 16 |
-
bonus = 0.
|
| 17 |
return round(base + bonus, 3)
|
| 18 |
else:
|
| 19 |
-
# penalize overconfidence when wrong
|
| 20 |
penalty = 0.3 * confidence
|
| 21 |
-
return round(max(0.
|
| 22 |
|
| 23 |
elif difficulty == "hard":
|
| 24 |
if correct:
|
| 25 |
-
# correct but penalize overconfidence (hard task, be humble)
|
| 26 |
base = 0.5
|
| 27 |
-
calibration_bonus = 0.
|
| 28 |
return round(base + calibration_bonus, 3)
|
| 29 |
else:
|
| 30 |
if confidence < 0.4:
|
| 31 |
-
return 0.15
|
| 32 |
else:
|
| 33 |
-
return 0.
|
|
|
|
| 5 |
|
| 6 |
if difficulty == "easy":
|
| 7 |
if correct:
|
| 8 |
+
return 0.95 # was 1.0
|
| 9 |
else:
|
| 10 |
+
return 0.05 # was 0.0
|
| 11 |
|
| 12 |
elif difficulty == "medium":
|
| 13 |
if correct:
|
|
|
|
| 14 |
base = 0.6
|
| 15 |
+
bonus = 0.35 * confidence # max = 0.95
|
| 16 |
return round(base + bonus, 3)
|
| 17 |
else:
|
|
|
|
| 18 |
penalty = 0.3 * confidence
|
| 19 |
+
return round(max(0.05, 0.2 - penalty), 3)
|
| 20 |
|
| 21 |
elif difficulty == "hard":
|
| 22 |
if correct:
|
|
|
|
| 23 |
base = 0.5
|
| 24 |
+
calibration_bonus = 0.45 * (1 - abs(confidence - 0.7))
|
| 25 |
return round(base + calibration_bonus, 3)
|
| 26 |
else:
|
| 27 |
if confidence < 0.4:
|
| 28 |
+
return 0.15
|
| 29 |
else:
|
| 30 |
+
return 0.05 # was 0.0
|