Initial commit

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -0,0 +1,135 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+tags:
+- mteb
+model-index:
+- name: fin-mpnet-base-v0.1
+  results:
+  - task:
+      type: Classification
+    dataset:
+      type: mteb/banking77
+      name: MTEB Banking77Classification
+      config: default
+      split: test
+      revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
+    metrics:
+    - type: accuracy
+      value: 80.25
+    - type: f1
+      value: 79.64999520103544
+  - task:
+      type: Retrieval
+    dataset:
+      type: fiqa
+      name: MTEB FiQA2018
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 37.747
+    - type: map_at_10
+      value: 72.223
+    - type: map_at_100
+      value: 73.802
+    - type: map_at_1000
+      value: 73.80499999999999
+    - type: map_at_3
+      value: 61.617999999999995
+    - type: map_at_5
+      value: 67.92200000000001
+    - type: mrr_at_1
+      value: 71.914
+    - type: mrr_at_10
+      value: 80.71000000000001
+    - type: mrr_at_100
+      value: 80.901
+    - type: mrr_at_1000
+      value: 80.901
+    - type: mrr_at_3
+      value: 78.935
+    - type: mrr_at_5
+      value: 80.193
+    - type: ndcg_at_1
+      value: 71.914
+    - type: ndcg_at_10
+      value: 79.912
+    - type: ndcg_at_100
+      value: 82.675
+    - type: ndcg_at_1000
+      value: 82.702
+    - type: ndcg_at_3
+      value: 73.252
+    - type: ndcg_at_5
+      value: 76.36
+    - type: precision_at_1
+      value: 71.914
+    - type: precision_at_10
+      value: 23.071
+    - type: precision_at_100
+      value: 2.62
+    - type: precision_at_1000
+      value: 0.263
+    - type: precision_at_3
+      value: 51.235
+    - type: precision_at_5
+      value: 38.117000000000004
+    - type: recall_at_1
+      value: 37.747
+    - type: recall_at_10
+      value: 91.346
+    - type: recall_at_100
+      value: 99.776
+    - type: recall_at_1000
+      value: 99.897
+    - type: recall_at_3
+      value: 68.691
+    - type: recall_at_5
+      value: 80.742
+---
+---
+
+v0.1 - full evaluation not complete
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+## Evaluation Results
+
+Model was evaluated during training only on the new finance QA examples, as such only financial relevant benchmarks were evaluated on for v0.1 [FiQA-2018,  BankingClassification77]
+
+The model currently shows the highest FiQA Retrieval score on the test set, on the MTEB Leaderboard (https://huggingface.co/spaces/mteb/leaderboard)
+
+The model will have likely suffered some performance on other benchmarks, i.e. BankingClassification77 has dropped from 81.6 to 80.25, this will be addressed for v0.2 and full evaluation on all sets will be run.
+
+
+## Training
+
+"sentence-transformers/all-mpnet-base-v2" was fine-tuned on 150k financial document QA examples using MNR Loss.

config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "/root/.cache/torch/sentence_transformers/sentence-transformers_all-mpnet-base-v2/",
+  "architectures": [
+    "MPNetModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "mpnet",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "relative_attention_num_buckets": 32,
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.2",
+  "vocab_size": 30527
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.0.0",
+    "transformers": "4.6.1",
+    "pytorch": "1.8.1"
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27a9f9dacab04c4d7ff8f528cb3515a8c8e21ff5d0ac2ddc9f9250efc554ec22
+size 437967672

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

mteb_metadata.md

@@ -0,0 +1,89 @@
+---
+tags:
+- mteb
+model-index:
+- name: fin-mpnet-base-v0.1
+  results:
+  - task:
+      type: Classification
+    dataset:
+      type: mteb/banking77
+      name: MTEB Banking77Classification
+      config: default
+      split: test
+      revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
+    metrics:
+    - type: accuracy
+      value: 80.25
+    - type: f1
+      value: 79.64999520103544
+  - task:
+      type: Retrieval
+    dataset:
+      type: fiqa
+      name: MTEB FiQA2018
+      config: default
+      split: test
+      revision: None
+    metrics:
+    - type: map_at_1
+      value: 37.747
+    - type: map_at_10
+      value: 72.223
+    - type: map_at_100
+      value: 73.802
+    - type: map_at_1000
+      value: 73.80499999999999
+    - type: map_at_3
+      value: 61.617999999999995
+    - type: map_at_5
+      value: 67.92200000000001
+    - type: mrr_at_1
+      value: 71.914
+    - type: mrr_at_10
+      value: 80.71000000000001
+    - type: mrr_at_100
+      value: 80.901
+    - type: mrr_at_1000
+      value: 80.901
+    - type: mrr_at_3
+      value: 78.935
+    - type: mrr_at_5
+      value: 80.193
+    - type: ndcg_at_1
+      value: 71.914
+    - type: ndcg_at_10
+      value: 79.912
+    - type: ndcg_at_100
+      value: 82.675
+    - type: ndcg_at_1000
+      value: 82.702
+    - type: ndcg_at_3
+      value: 73.252
+    - type: ndcg_at_5
+      value: 76.36
+    - type: precision_at_1
+      value: 71.914
+    - type: precision_at_10
+      value: 23.071
+    - type: precision_at_100
+      value: 2.62
+    - type: precision_at_1000
+      value: 0.263
+    - type: precision_at_3
+      value: 51.235
+    - type: precision_at_5
+      value: 38.117000000000004
+    - type: recall_at_1
+      value: 37.747
+    - type: recall_at_10
+      value: 91.346
+    - type: recall_at_100
+      value: 99.776
+    - type: recall_at_1000
+      value: 99.897
+    - type: recall_at_3
+      value: 68.691
+    - type: recall_at_5
+      value: 80.742
+---

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 384,
+  "do_lower_case": false
+}

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,72 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "104": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30526": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "do_lower_case": true,
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "max_length": 128,
+  "model_max_length": 512,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "</s>",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "MPNetTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}