Upload create-submission-result 1.ipynb

Operational_Instructions/create-submission-result 1.ipynb

@@ -0,0 +1,132 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5e875f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import jsonschema\n",
+    "from jsonschema import validate\n",
+    "import json\n",
+    "\n",
+    "# LiveRAG Answer JSON schema: \n",
+    "json_schema = \"\"\"\n",
+    "{ \n",
+    "\"$schema\": \"http://json-schema.org/draft-07/schema#\", \n",
+    "\n",
+    "  \"title\": \"Answer file schema\", \n",
+    "  \"type\": \"object\", \n",
+    "  \"properties\": { \n",
+    "    \"id\": { \n",
+    "      \"type\": \"integer\", \n",
+    "      \"description\": \"Question ID\" \n",
+    "    }, \n",
+    "    \"question\": { \n",
+    "      \"type\": \"string\", \n",
+    "      \"description\": \"The question\" \n",
+    "    }, \n",
+    "    \"passages\": { \n",
+    "      \"type\": \"array\", \n",
+    "      \"description\": \"Passages used and related FineWeb doc IDs, ordered by decreasing importance\", \n",
+    "      \"items\": { \n",
+    "        \"type\": \"object\", \n",
+    "        \"properties\": {\n",
+    "          \"passage\": { \n",
+    "            \"type\": \"string\", \n",
+    "            \"description\": \"Passage text\" \n",
+    "          }, \n",
+    "          \"doc_IDs\": {\n",
+    "            \"type\": \"array\", \n",
+    "            \"description\": \"Passage related FineWeb doc IDs, ordered by decreasing importance\", \n",
+    "            \"items\": { \n",
+    "              \"type\": \"string\", \n",
+    "              \"description\": \"FineWeb doc ID, e.g., <urn:uuid:d69cbebc-133a-4ebe-9378-68235ec9f091>\"\n",
+    "            } \n",
+    "          } \n",
+    "        },\n",
+    "        \"required\": [\"passage\", \"doc_IDs\"]\n",
+    "      }\n",
+    "    }, \n",
+    "    \"final_prompt\": {\n",
+    "      \"type\": \"string\",\n",
+    "      \"description\": \"Final prompt, as submitted to Falcon LLM\"\n",
+    "    },\n",
+    "    \"answer\": {\n",
+    "      \"type\": \"string\",\n",
+    "      \"description\": \"Your answer\"\n",
+    "    }\n",
+    "  },\n",
+    "  \"required\": [\"id\", \"question\", \"passages\", \"final_prompt\", \"answer\"]\n",
+    "}\n",
+    "\"\"\"\n",
+    "\n",
+    "# Code that generates the output\n",
+    "answers = pd.DataFrame({\n",
+    "    \"id\": [1, 2],\n",
+    "    \"question\": [\"What is the capital of France?\", \"What is the capital of Germany?\"],\n",
+    "    \"passages\": [\n",
+    "        [\n",
+    "            {\"passage\": \"Paris is the capital and most populous city of France.\",\n",
+    "             \"doc_IDs\": [\"<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>\", \"<urn:uuid:1234abcd-5678-efgh-9202-ijklmnopqrst>\"]},\n",
+    "            {\"passage\": \"France is located in Western Europe.\",\n",
+    "             \"doc_IDs\": [\"<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>\"]}\n",
+    "        ],\n",
+    "        [\n",
+    "            {\"passage\": \"Berlin is the capital of Germany.\",\n",
+    "             \"doc_IDs\": [\"<urn:uuid:1234abcd-5678-efgh-9101-ijklmnopqrst>\"]}\n",
+    "        ]\n",
+    "    ],\n",
+    "    \"final_prompt\": [\n",
+    "        \"Using the following - Paris is the capital and most populous city of France - and - France is located in Western Europe - answer the question: What is the capital of France?\",\n",
+    "        \"Using the following - Berlin is the capital of Germany - answer the question: What is the capital of Germany?\"\n",
+    "    ],\n",
+    "    \"answer\": [\"Paris\", \"Berlin\"]\n",
+    "})\n",
+    "\n",
+    "# Convert to JSON format\n",
+    "answers_json = answers.to_json(orient='records', lines=True, force_ascii=False)\n",
+    "\n",
+    "# Or just save to a file\n",
+    "answers.to_json(\"answers.jsonl\", orient='records', lines=True, force_ascii=False)\n",
+    "\n",
+    "# Load the file to make sure it is ok\n",
+    "loaded_answers = pd.read_json(\"answers.jsonl\", lines=True)\n",
+    "\n",
+    "# Load the JSON schema\n",
+    "schema = json.loads(json_schema)\n",
+    "\n",
+    "# Validate each Answer JSON object against the schema\n",
+    "for answer in loaded_answers.to_dict(orient='records'):\n",
+    "    try:\n",
+    "        validate(instance=answer, schema=schema)\n",
+    "        print(f\"Answer {answer['id']} is valid.\")\n",
+    "    except jsonschema.exceptions.ValidationError as e:\n",
+    "        print(f\"Answer {answer['id']} is invalid: {e.message}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}