From a94e970cabfebff76834a4306acb2d9843dc59d2 Mon Sep 17 00:00:00 2001 From: lintool Date: Tue, 10 Sep 2024 15:28:22 -0400 Subject: [PATCH] Initial addition of tolerance scores in yaml --- src/main/python/run_regression.py | 37 +++++++++++-------- ....0-arguana.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...0.0-bioasq.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...mate-fever.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...ck-android.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...ck-english.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...ack-gaming.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...pstack-gis.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...athematica.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...ck-physics.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...rogrammers.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...tack-stats.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...pstack-tex.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...stack-unix.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...webmasters.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...-wordpress.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...dia-entity.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ....0.0-fever.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...1.0.0-fiqa.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...0-hotpotqa.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...0-nfcorpus.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...-v1.0.0-nq.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ....0.0-quora.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...0-robust04.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ....0-scidocs.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ....0-scifact.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...0-signal1m.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...trec-covid.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...-trec-news.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ ...touche2020.bge-base-en-v1.5.flat.onnx.yaml | 7 ++++ 30 files changed, 224 insertions(+), 16 deletions(-) diff --git a/src/main/python/run_regression.py b/src/main/python/run_regression.py index ec66fe657..f0f87874e 100644 --- a/src/main/python/run_regression.py +++ b/src/main/python/run_regression.py @@ -393,22 +393,27 @@ def evaluate_and_verify(yaml_data, dry_run): using_flat = True if 'type' in model and model['type'] == 'flat' else False if using_flat: - # Extract model - match = flat_model_type_pattern.search(model['name']) - model_type = match.group(1) - - if 'BEIR' in topic_set['name']: - # Extract BEIR dataset - match = beir_dataset_pattern.search(topic_set['name']) - beir_dataset = match.group(1) - - tolerance_ok = beir_flat_tolerance[model_type][beir_dataset] - elif 'MS MARCO Passage' in topic_set['name']: - tolerance_ok = msmarco_v1_flat_tolerance[model_type][model['name']] - elif 'DL19' in topic_set['name']: - tolerance_ok = dl19_flat_tolerance[model_type][model['name']] - elif using_flat and 'DL20' in topic_set['name']: - tolerance_ok = dl20_flat_tolerance[model_type][model['name']] + if 'tolerance' in model: + #print(model['tolerance']) + #print(metric) + tolerance_ok = model['tolerance'][metric['metric']][i] + else: + # Extract model + match = flat_model_type_pattern.search(model['name']) + model_type = match.group(1) + + if 'BEIR' in topic_set['name']: + # Extract BEIR dataset + match = beir_dataset_pattern.search(topic_set['name']) + beir_dataset = match.group(1) + + tolerance_ok = beir_flat_tolerance[model_type][beir_dataset] + elif 'MS MARCO Passage' in topic_set['name']: + tolerance_ok = msmarco_v1_flat_tolerance[model_type][model['name']] + elif 'DL19' in topic_set['name']: + tolerance_ok = dl19_flat_tolerance[model_type][model['name']] + elif using_flat and 'DL20' in topic_set['name']: + tolerance_ok = dl20_flat_tolerance[model_type][model['name']] if using_hnsw: # Extract model diff --git a/src/main/resources/regression/beir-v1.0.0-arguana.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-arguana.bge-base-en-v1.5.flat.onnx.yaml index 84800cd0e..af11b0460 100644 --- a/src/main/resources/regression/beir-v1.0.0-arguana.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-arguana.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.9915 R@1000: - 0.9964 + tolerance: + nDCG@10: + - 0.02 + R@100: + - 0.02 + R@1000: + - 0.004 diff --git a/src/main/resources/regression/beir-v1.0.0-bioasq.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-bioasq.bge-base-en-v1.5.flat.onnx.yaml index eccff084e..1d84baaf2 100644 --- a/src/main/resources/regression/beir-v1.0.0-bioasq.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-bioasq.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.6317 R@1000: - 0.8059 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0002 + R@1000: + - 0.0004 diff --git a/src/main/resources/regression/beir-v1.0.0-climate-fever.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-climate-fever.bge-base-en-v1.5.flat.onnx.yaml index 4ed408088..ed436e517 100644 --- a/src/main/resources/regression/beir-v1.0.0-climate-fever.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-climate-fever.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.6362 R@1000: - 0.8307 + tolerance: + nDCG@10: + - 0.0003 + R@100: + - 0.001 + R@1000: + - 0.0002 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android.bge-base-en-v1.5.flat.onnx.yaml index 303dd14d9..e5355f4f8 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.8454 R@1000: - 0.9611 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english.bge-base-en-v1.5.flat.onnx.yaml index c2ed6c5e1..e0c47dfb6 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.7587 R@1000: - 0.8839 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0002 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming.bge-base-en-v1.5.flat.onnx.yaml index f38649349..72340811e 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gaming.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.9036 R@1000: - 0.9719 + tolerance: + nDCG@10: + - 0.0003 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis.bge-base-en-v1.5.flat.onnx.yaml index dcfbfb588..2a82af70d 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.7682 R@1000: - 0.9117 + tolerance: + nDCG@10: + - 0.0005 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica.bge-base-en-v1.5.flat.onnx.yaml index ae8190d90..7c722e3ad 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-mathematica.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.6922 R@1000: - 0.8810 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics.bge-base-en-v1.5.flat.onnx.yaml index fdd48a873..ad5f9c690 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.8081 R@1000: - 0.9406 + tolerance: + nDCG@10: + - 0.0003 + R@100: + - 0.0004 + R@1000: + - 0.001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers.bge-base-en-v1.5.flat.onnx.yaml index a1be8a156..0c814ac06 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.7856 R@1000: - 0.9348 + tolerance: + nDCG@10: + - 0.0005 + R@100: + - 0.0001 + R@1000: + - 0.0006 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats.bge-base-en-v1.5.flat.onnx.yaml index f359fddf8..fa0db8bf6 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.6727 R@1000: - 0.8445 + tolerance: + nDCG@10: + - 0.0005 + R@100: + - 0.0009 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex.bge-base-en-v1.5.flat.onnx.yaml index cb3c2f077..08c00ffa5 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.6486 R@1000: - 0.8537 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0004 + R@1000: + - 0.0002 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix.bge-base-en-v1.5.flat.onnx.yaml index 25a50d459..e3de3d860 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.7797 R@1000: - 0.9237 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0001 + R@1000: + - 0.0002 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters.bge-base-en-v1.5.flat.onnx.yaml index 88dc49625..2f70d1c42 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.7774 R@1000: - 0.9380 + tolerance: + nDCG@10: + - 0.0008 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress.bge-base-en-v1.5.flat.onnx.yaml index 66939203d..97ee52402 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.7065 R@1000: - 0.8861 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.002 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity.bge-base-en-v1.5.flat.onnx.yaml index c9e7c828e..2aeab66ec 100644 --- a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.5303 R@1000: - 0.7833 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0006 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-fever.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-fever.bge-base-en-v1.5.flat.onnx.yaml index 96947b503..528055897 100644 --- a/src/main/resources/regression/beir-v1.0.0-fever.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-fever.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.9719 R@1000: - 0.9855 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-fiqa.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-fiqa.bge-base-en-v1.5.flat.onnx.yaml index aa626889e..88f247f00 100644 --- a/src/main/resources/regression/beir-v1.0.0-fiqa.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-fiqa.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.7415 R@1000: - 0.9083 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-hotpotqa.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-hotpotqa.bge-base-en-v1.5.flat.onnx.yaml index e87401a60..d428e8bb7 100644 --- a/src/main/resources/regression/beir-v1.0.0-hotpotqa.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-hotpotqa.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.8727 R@1000: - 0.9424 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0002 + R@1000: + - 0.0002 diff --git a/src/main/resources/regression/beir-v1.0.0-nfcorpus.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-nfcorpus.bge-base-en-v1.5.flat.onnx.yaml index 25d096dba..e02719206 100644 --- a/src/main/resources/regression/beir-v1.0.0-nfcorpus.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-nfcorpus.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.3368 R@1000: - 0.6622 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-nq.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-nq.bge-base-en-v1.5.flat.onnx.yaml index 4dac6d4d5..e44641519 100644 --- a/src/main/resources/regression/beir-v1.0.0-nq.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-nq.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.9415 R@1000: - 0.9859 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0002 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-quora.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-quora.bge-base-en-v1.5.flat.onnx.yaml index 8ae6725dd..9bdf069d1 100644 --- a/src/main/resources/regression/beir-v1.0.0-quora.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-quora.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.9967 R@1000: - 0.9998 + tolerance: + nDCG@10: + - 0.002 + R@100: + - 0.0002 + R@1000: + - 0.0002 diff --git a/src/main/resources/regression/beir-v1.0.0-robust04.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-robust04.bge-base-en-v1.5.flat.onnx.yaml index dee9b35ad..88404155f 100644 --- a/src/main/resources/regression/beir-v1.0.0-robust04.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-robust04.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.3507 R@1000: - 0.5981 + tolerance: + nDCG@10: + - 0.004 + R@100: + - 0.0004 + R@1000: + - 0.003 diff --git a/src/main/resources/regression/beir-v1.0.0-scidocs.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-scidocs.bge-base-en-v1.5.flat.onnx.yaml index 59da611fa..da5618cd4 100644 --- a/src/main/resources/regression/beir-v1.0.0-scidocs.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-scidocs.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.4959 R@1000: - 0.7824 + tolerance: + nDCG@10: + - 0.0003 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-scifact.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-scifact.bge-base-en-v1.5.flat.onnx.yaml index 77768ac35..0dda4f64a 100644 --- a/src/main/resources/regression/beir-v1.0.0-scifact.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-scifact.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.9667 R@1000: - 0.9967 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-signal1m.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-signal1m.bge-base-en-v1.5.flat.onnx.yaml index af4a50248..7b24d4b2a 100644 --- a/src/main/resources/regression/beir-v1.0.0-signal1m.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-signal1m.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.3112 R@1000: - 0.5331 + tolerance: + nDCG@10: + - 0.0001 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-covid.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-trec-covid.bge-base-en-v1.5.flat.onnx.yaml index 9691015a9..c24b8d1a5 100644 --- a/src/main/resources/regression/beir-v1.0.0-trec-covid.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-trec-covid.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.1406 R@1000: - 0.4768 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0001 + R@1000: + - 0.0004 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-news.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-trec-news.bge-base-en-v1.5.flat.onnx.yaml index 213a7ee58..053584665 100644 --- a/src/main/resources/regression/beir-v1.0.0-trec-news.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-trec-news.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.4992 R@1000: - 0.7875 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.0001 + R@1000: + - 0.0001 diff --git a/src/main/resources/regression/beir-v1.0.0-webis-touche2020.bge-base-en-v1.5.flat.onnx.yaml b/src/main/resources/regression/beir-v1.0.0-webis-touche2020.bge-base-en-v1.5.flat.onnx.yaml index 33d4bf6e7..cb70d3a94 100644 --- a/src/main/resources/regression/beir-v1.0.0-webis-touche2020.bge-base-en-v1.5.flat.onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-webis-touche2020.bge-base-en-v1.5.flat.onnx.yaml @@ -51,3 +51,10 @@ models: - 0.4857 R@1000: - 0.8298 + tolerance: + nDCG@10: + - 0.0002 + R@100: + - 0.002 + R@1000: + - 0.0001