Initial addition of tolerance scores in yaml

src/main/python/run_regression.py

@@ -393,22 +393,27 @@ def evaluate_and_verify(yaml_data, dry_run):
                 using_flat = True if 'type' in model and model['type'] == 'flat' else False
 
                 if using_flat:
-                    # Extract model
-                    match = flat_model_type_pattern.search(model['name'])
-                    model_type = match.group(1)
-
-                    if 'BEIR' in topic_set['name']:
-                        # Extract BEIR dataset
-                        match = beir_dataset_pattern.search(topic_set['name'])
-                        beir_dataset = match.group(1)
-
-                        tolerance_ok = beir_flat_tolerance[model_type][beir_dataset]
-                    elif 'MS MARCO Passage' in topic_set['name']:
-                        tolerance_ok = msmarco_v1_flat_tolerance[model_type][model['name']]
-                    elif 'DL19' in topic_set['name']:
-                        tolerance_ok = dl19_flat_tolerance[model_type][model['name']]
-                    elif using_flat and 'DL20' in topic_set['name']:
-                        tolerance_ok = dl20_flat_tolerance[model_type][model['name']]
+                    if 'tolerance' in model:
+                        #print(model['tolerance'])
+                        #print(metric)
+                        tolerance_ok = model['tolerance'][metric['metric']][i]
+                    else:
+                        # Extract model
+                        match = flat_model_type_pattern.search(model['name'])
+                        model_type = match.group(1)
+
+                        if 'BEIR' in topic_set['name']:
+                            # Extract BEIR dataset
+                            match = beir_dataset_pattern.search(topic_set['name'])
+                            beir_dataset = match.group(1)
+
+                            tolerance_ok = beir_flat_tolerance[model_type][beir_dataset]
+                        elif 'MS MARCO Passage' in topic_set['name']:
+                            tolerance_ok = msmarco_v1_flat_tolerance[model_type][model['name']]
+                        elif 'DL19' in topic_set['name']:
+                            tolerance_ok = dl19_flat_tolerance[model_type][model['name']]
+                        elif using_flat and 'DL20' in topic_set['name']:
+                            tolerance_ok = dl20_flat_tolerance[model_type][model['name']]
 
                 if using_hnsw:
                     # Extract model

src/main/resources/regression/beir-v1.0.0-arguana.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9915
       R@1000:
         - 0.9964
+    tolerance:
+      nDCG@10:
+        - 0.02
+      R@100:
+        - 0.02
+      R@1000:
+        - 0.004

src/main/resources/regression/beir-v1.0.0-bioasq.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6317
       R@1000:
         - 0.8059
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0002
+      R@1000:
+        - 0.0004

src/main/resources/regression/beir-v1.0.0-climate-fever.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6362
       R@1000:
         - 0.8307
+    tolerance:
+      nDCG@10:
+        - 0.0003
+      R@100:
+        - 0.001
+      R@1000:
+        - 0.0002

src/main/resources/regression/beir-v1.0.0-cqadupstack-android.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.8454
       R@1000:
         - 0.9611
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-cqadupstack-english.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7587
       R@1000:
         - 0.8839
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0002
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9036
       R@1000:
         - 0.9719
+    tolerance:
+      nDCG@10:
+        - 0.0003
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-cqadupstack-gis.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7682
       R@1000:
         - 0.9117
+    tolerance:
+      nDCG@10:
+        - 0.0005
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6922
       R@1000:
         - 0.8810
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-cqadupstack-physics.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.8081
       R@1000:
         - 0.9406
+    tolerance:
+      nDCG@10:
+        - 0.0003
+      R@100:
+        - 0.0004
+      R@1000:
+        - 0.001

src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7856
       R@1000:
         - 0.9348
+    tolerance:
+      nDCG@10:
+        - 0.0005
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0006

src/main/resources/regression/beir-v1.0.0-cqadupstack-stats.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6727
       R@1000:
         - 0.8445
+    tolerance:
+      nDCG@10:
+        - 0.0005
+      R@100:
+        - 0.0009
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-cqadupstack-tex.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.6486
       R@1000:
         - 0.8537
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0004
+      R@1000:
+        - 0.0002

src/main/resources/regression/beir-v1.0.0-cqadupstack-unix.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7797
       R@1000:
         - 0.9237
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0002

src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7774
       R@1000:
         - 0.9380
+    tolerance:
+      nDCG@10:
+        - 0.0008
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7065
       R@1000:
         - 0.8861
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.002
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-dbpedia-entity.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.5303
       R@1000:
         - 0.7833
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0006
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-fever.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9719
       R@1000:
         - 0.9855
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-fiqa.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.7415
       R@1000:
         - 0.9083
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-hotpotqa.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.8727
       R@1000:
         - 0.9424
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0002
+      R@1000:
+        - 0.0002

src/main/resources/regression/beir-v1.0.0-nfcorpus.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.3368
       R@1000:
         - 0.6622
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-nq.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9415
       R@1000:
         - 0.9859
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0002
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-quora.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9967
       R@1000:
         - 0.9998
+    tolerance:
+      nDCG@10:
+        - 0.002
+      R@100:
+        - 0.0002
+      R@1000:
+        - 0.0002

src/main/resources/regression/beir-v1.0.0-robust04.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.3507
       R@1000:
         - 0.5981
+    tolerance:
+      nDCG@10:
+        - 0.004
+      R@100:
+        - 0.0004
+      R@1000:
+        - 0.003

src/main/resources/regression/beir-v1.0.0-scidocs.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.4959
       R@1000:
         - 0.7824
+    tolerance:
+      nDCG@10:
+        - 0.0003
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-scifact.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.9667
       R@1000:
         - 0.9967
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-signal1m.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.3112
       R@1000:
         - 0.5331
+    tolerance:
+      nDCG@10:
+        - 0.0001
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-trec-covid.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.1406
       R@1000:
         - 0.4768
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0004

src/main/resources/regression/beir-v1.0.0-trec-news.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.4992
       R@1000:
         - 0.7875
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.0001
+      R@1000:
+        - 0.0001

src/main/resources/regression/beir-v1.0.0-webis-touche2020.bge-base-en-v1.5.flat.onnx.yaml

@@ -51,3 +51,10 @@ models:
         - 0.4857
       R@1000:
         - 0.8298
+    tolerance:
+      nDCG@10:
+        - 0.0002
+      R@100:
+        - 0.002
+      R@1000:
+        - 0.0001