From 2106a87c5ad08d0e04449ad0835916ecc066bf60 Mon Sep 17 00:00:00 2001
From: Ivan Pavlovich <ivan.pavlovic@hes-so.ch>
Date: Fri, 14 Mar 2025 14:51:52 +0100
Subject: [PATCH] Recalculated tokens and LLM price with real input/output
prompts
---
models/LLM/Ollama/ollama_wrapper.py | 2 +-
models/LLM/Tokenizer/data/token_count.json | 426 +++++++++---------
models/LLM/Tokenizer/token_count.py | 19 +-
models/LLM/prices/data/llm_prices.json | 384 ++++++++--------
testModel/__pycache__/metrics.cpython-313.pyc | Bin 1652 -> 1947 bytes
testModel/__pycache__/utils.cpython-313.pyc | Bin 4428 -> 4489 bytes
testModel/launch_tests.py | 25 +
7 files changed, 447 insertions(+), 409 deletions(-)
create mode 100644 testModel/launch_tests.py
diff --git a/models/LLM/Ollama/ollama_wrapper.py b/models/LLM/Ollama/ollama_wrapper.py
index 3e931bb66..fbcd37102 100644
--- a/models/LLM/Ollama/ollama_wrapper.py
+++ b/models/LLM/Ollama/ollama_wrapper.py
@@ -19,7 +19,7 @@ def classify(model, sequence, labels):
prompt += 'text: ' + sequence + '\n'
prompt += 'labels: ' + "[" + ", ".join(labels) + ']\n'
prompt += 'Give the response in json format: { "labels": [] }. Follow this writing to the letter and don t add text around it. Only put the labels that you select between [].'
- prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between \'\'.'
+ prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between "".'
response: ChatResponse = chat(model=model, messages=[
{
diff --git a/models/LLM/Tokenizer/data/token_count.json b/models/LLM/Tokenizer/data/token_count.json
index fd0f4dd3b..97e080c23 100644
--- a/models/LLM/Tokenizer/data/token_count.json
+++ b/models/LLM/Tokenizer/data/token_count.json
@@ -1,41 +1,41 @@
{
"bert-base-uncased": {
- "ALL": 62940760,
+ "ALL": 89156232,
"NO KEYWORDS": {
"day": {
"input": {
"min": 0,
- "max": 336390,
- "mean": 55947.34222222222
+ "max": 478958,
+ "mean": 79249.984
},
"output": {
"min": 0,
- "max": 100400,
- "mean": 16410.31111111111
+ "max": 47188,
+ "mean": 7712.846222222222
}
},
"week": {
"input": {
"min": 0,
- "max": 610773,
- "mean": 390936.39751552796
+ "max": 868077,
+ "mean": 553765.4161490683
},
"output": {
"min": 0,
- "max": 181200,
- "mean": 114668.32298136647
+ "max": 85164,
+ "mean": 53894.11180124224
}
},
"month": {
"input": {
- "min": 149220,
- "max": 1988608,
- "mean": 1701101.6216216215
+ "min": 206588,
+ "max": 2817488,
+ "mean": 2409627.891891892
},
"output": {
- "min": 40400,
- "max": 587200,
- "mean": 498962.1621621622
+ "min": 18988,
+ "max": 275984,
+ "mean": 234512.2162162162
}
}
},
@@ -43,37 +43,37 @@
"day": {
"input": {
"min": 0,
- "max": 14061,
- "mean": 2494.1857777777777
+ "max": 19599,
+ "mean": 3528.703111111111
},
"output": {
"min": 0,
- "max": 3900,
- "mean": 728.5333333333333
+ "max": 1833,
+ "mean": 342.41066666666666
}
},
"week": {
"input": {
"min": 0,
- "max": 28111,
- "mean": 17428.316770186335
+ "max": 38477,
+ "mean": 24657.08695652174
},
"output": {
"min": 0,
- "max": 8300,
- "mean": 5090.683229813664
+ "max": 3901,
+ "mean": 2392.6211180124224
}
},
"month": {
"input": {
- "min": 12058,
- "max": 105204,
- "mean": 75836.72972972973
+ "min": 16460,
+ "max": 147094,
+ "mean": 107291.64864864865
},
"output": {
- "min": 3100,
- "max": 29500,
- "mean": 22151.35135135135
+ "min": 1457,
+ "max": 13865,
+ "mean": 10411.135135135135
}
}
},
@@ -81,37 +81,37 @@
"day": {
"input": {
"min": 0,
- "max": 14061,
- "mean": 2494.1857777777777
+ "max": 19599,
+ "mean": 3528.703111111111
},
"output": {
"min": 0,
- "max": 3900,
- "mean": 728.5333333333333
+ "max": 1833,
+ "mean": 342.41066666666666
}
},
"week": {
"input": {
"min": 0,
- "max": 28111,
- "mean": 17428.316770186335
+ "max": 38477,
+ "mean": 24657.08695652174
},
"output": {
"min": 0,
- "max": 8300,
- "mean": 5090.683229813664
+ "max": 3901,
+ "mean": 2392.6211180124224
}
},
"month": {
"input": {
- "min": 12058,
- "max": 105204,
- "mean": 75836.72972972973
+ "min": 16460,
+ "max": 147094,
+ "mean": 107291.64864864865
},
"output": {
- "min": 3100,
- "max": 29500,
- "mean": 22151.35135135135
+ "min": 1457,
+ "max": 13865,
+ "mean": 10411.135135135135
}
}
},
@@ -119,37 +119,37 @@
"day": {
"input": {
"min": 0,
- "max": 17409,
- "mean": 3292.2702222222224
+ "max": 24083,
+ "mean": 4632.119111111111
},
"output": {
"min": 0,
- "max": 4700,
- "mean": 943.5555555555555
+ "max": 2209,
+ "mean": 443.4711111111111
}
},
"week": {
"input": {
"min": 0,
- "max": 36705,
- "mean": 23004.993788819876
+ "max": 51710,
+ "mean": 32367.291925465837
},
"output": {
"min": 0,
- "max": 11200,
- "mean": 6593.167701863354
+ "max": 5264,
+ "mean": 3098.7888198757764
}
},
"month": {
"input": {
- "min": 13250,
- "max": 124682,
- "mean": 100102.81081081081
+ "min": 18078,
+ "max": 173956,
+ "mean": 140841.45945945947
},
"output": {
- "min": 3400,
- "max": 34700,
- "mean": 28689.18918918919
+ "min": 1598,
+ "max": 16309,
+ "mean": 13483.918918918918
}
}
},
@@ -157,78 +157,78 @@
"day": {
"input": {
"min": 0,
- "max": 24471,
- "mean": 4493.711111111111
+ "max": 33985,
+ "mean": 6335.672
},
"output": {
"min": 0,
- "max": 6700,
- "mean": 1297.1555555555556
+ "max": 3149,
+ "mean": 609.6631111111111
}
},
"week": {
"input": {
"min": 0,
- "max": 49793,
- "mean": 31400.155279503106
+ "max": 71519,
+ "mean": 44271
},
"output": {
"min": 0,
- "max": 15300,
- "mean": 9063.975155279502
+ "max": 7191,
+ "mean": 4260.068322981367
}
},
"month": {
"input": {
- "min": 17661,
- "max": 172341,
- "mean": 136633.1081081081
+ "min": 24193,
+ "max": 240785,
+ "mean": 192638.67567567568
},
"output": {
- "min": 4600,
- "max": 48200,
- "mean": 39440.54054054054
+ "min": 2162,
+ "max": 22654,
+ "mean": 18537.054054054053
}
}
}
},
"roberta-base": {
- "ALL": 61502461,
+ "ALL": 87109722,
"NO KEYWORDS": {
"day": {
"input": {
"min": 0,
- "max": 327932,
- "mean": 54668.854222222224
+ "max": 467190,
+ "mean": 77430.864
},
"output": {
"min": 0,
- "max": 100400,
- "mean": 16410.31111111111
+ "max": 43172,
+ "mean": 7056.433777777778
}
},
"week": {
"input": {
"min": 0,
- "max": 597010,
- "mean": 382002.8633540373
+ "max": 848340,
+ "mean": 541054.1739130435
},
"output": {
"min": 0,
- "max": 181200,
- "mean": 114668.32298136647
+ "max": 77916,
+ "mean": 49307.37888198758
}
},
"month": {
"input": {
- "min": 145722,
- "max": 1940671,
- "mean": 1662228.6756756757
+ "min": 201746,
+ "max": 2748787,
+ "mean": 2354316.810810811
},
"output": {
- "min": 40400,
- "max": 587200,
- "mean": 498962.1621621622
+ "min": 17372,
+ "max": 252496,
+ "mean": 214553.72972972973
}
}
},
@@ -236,37 +236,37 @@
"day": {
"input": {
"min": 0,
- "max": 13765,
- "mean": 2451.286222222222
+ "max": 19177,
+ "mean": 3461.84
},
"output": {
"min": 0,
- "max": 3900,
- "mean": 728.5333333333333
+ "max": 1677,
+ "mean": 313.26933333333335
}
},
"week": {
"input": {
"min": 0,
- "max": 27593,
- "mean": 17128.552795031057
+ "max": 37719,
+ "mean": 24189.875776397515
},
"output": {
"min": 0,
- "max": 8300,
- "mean": 5090.683229813664
+ "max": 3569,
+ "mean": 2188.9937888198756
}
},
"month": {
"input": {
- "min": 11886,
- "max": 103256,
- "mean": 74532.35135135135
+ "min": 16182,
+ "max": 144179,
+ "mean": 105258.64864864865
},
"output": {
- "min": 3100,
- "max": 29500,
- "mean": 22151.35135135135
+ "min": 1333,
+ "max": 12685,
+ "mean": 9525.081081081082
}
}
},
@@ -274,37 +274,37 @@
"day": {
"input": {
"min": 0,
- "max": 13765,
- "mean": 2451.286222222222
+ "max": 19177,
+ "mean": 3461.84
},
"output": {
"min": 0,
- "max": 3900,
- "mean": 728.5333333333333
+ "max": 1677,
+ "mean": 313.26933333333335
}
},
"week": {
"input": {
"min": 0,
- "max": 27593,
- "mean": 17128.552795031057
+ "max": 37719,
+ "mean": 24189.875776397515
},
"output": {
"min": 0,
- "max": 8300,
- "mean": 5090.683229813664
+ "max": 3569,
+ "mean": 2188.9937888198756
}
},
"month": {
"input": {
- "min": 11886,
- "max": 103256,
- "mean": 74532.35135135135
+ "min": 16182,
+ "max": 144179,
+ "mean": 105258.64864864865
},
"output": {
- "min": 3100,
- "max": 29500,
- "mean": 22151.35135135135
+ "min": 1333,
+ "max": 12685,
+ "mean": 9525.081081081082
}
}
},
@@ -312,37 +312,37 @@
"day": {
"input": {
"min": 0,
- "max": 17097,
- "mean": 3239.8124444444443
+ "max": 23618,
+ "mean": 4548.530666666667
},
"output": {
"min": 0,
- "max": 4700,
- "mean": 943.5555555555555
+ "max": 2021,
+ "mean": 405.7288888888889
}
},
"week": {
"input": {
"min": 0,
- "max": 36147,
- "mean": 22638.44099378882
+ "max": 50629,
+ "mean": 31783.211180124224
},
"output": {
"min": 0,
- "max": 11200,
- "mean": 6593.167701863354
+ "max": 4816,
+ "mean": 2835.0621118012423
}
},
"month": {
"input": {
- "min": 13077,
- "max": 122603,
- "mean": 98507.81081081081
+ "min": 17788,
+ "max": 170739,
+ "mean": 138299.9189189189
},
"output": {
- "min": 3400,
- "max": 34700,
- "mean": 28689.18918918919
+ "min": 1462,
+ "max": 14921,
+ "mean": 12336.351351351352
}
}
},
@@ -350,78 +350,78 @@
"day": {
"input": {
"min": 0,
- "max": 23993,
- "mean": 4422.054222222222
+ "max": 33290,
+ "mean": 6221.055111111111
},
"output": {
"min": 0,
- "max": 6700,
- "mean": 1297.1555555555556
+ "max": 2881,
+ "mean": 557.7768888888888
}
},
"week": {
"input": {
"min": 0,
- "max": 48994,
- "mean": 30899.447204968943
+ "max": 70202,
+ "mean": 43470.105590062114
},
"output": {
"min": 0,
- "max": 15300,
- "mean": 9063.975155279502
+ "max": 6579,
+ "mean": 3897.509316770186
}
},
"month": {
"input": {
- "min": 17429,
- "max": 169516,
- "mean": 134454.35135135136
+ "min": 23801,
+ "max": 236368,
+ "mean": 189153.7027027027
},
"output": {
- "min": 4600,
- "max": 48200,
- "mean": 39440.54054054054
+ "min": 1978,
+ "max": 20726,
+ "mean": 16959.432432432433
}
}
}
},
"facebook/bart-large": {
- "ALL": 61502461,
+ "ALL": 87109722,
"NO KEYWORDS": {
"day": {
"input": {
"min": 0,
- "max": 327932,
- "mean": 54668.854222222224
+ "max": 467190,
+ "mean": 77430.864
},
"output": {
"min": 0,
- "max": 100400,
- "mean": 16410.31111111111
+ "max": 43172,
+ "mean": 7056.433777777778
}
},
"week": {
"input": {
"min": 0,
- "max": 597010,
- "mean": 382002.8633540373
+ "max": 848340,
+ "mean": 541054.1739130435
},
"output": {
"min": 0,
- "max": 181200,
- "mean": 114668.32298136647
+ "max": 77916,
+ "mean": 49307.37888198758
}
},
"month": {
"input": {
- "min": 145722,
- "max": 1940671,
- "mean": 1662228.6756756757
+ "min": 201746,
+ "max": 2748787,
+ "mean": 2354316.810810811
},
"output": {
- "min": 40400,
- "max": 587200,
- "mean": 498962.1621621622
+ "min": 17372,
+ "max": 252496,
+ "mean": 214553.72972972973
}
}
},
@@ -429,37 +429,37 @@
"day": {
"input": {
"min": 0,
- "max": 13765,
- "mean": 2451.286222222222
+ "max": 19177,
+ "mean": 3461.84
},
"output": {
"min": 0,
- "max": 3900,
- "mean": 728.5333333333333
+ "max": 1677,
+ "mean": 313.26933333333335
}
},
"week": {
"input": {
"min": 0,
- "max": 27593,
- "mean": 17128.552795031057
+ "max": 37719,
+ "mean": 24189.875776397515
},
"output": {
"min": 0,
- "max": 8300,
- "mean": 5090.683229813664
+ "max": 3569,
+ "mean": 2188.9937888198756
}
},
"month": {
"input": {
- "min": 11886,
- "max": 103256,
- "mean": 74532.35135135135
+ "min": 16182,
+ "max": 144179,
+ "mean": 105258.64864864865
},
"output": {
- "min": 3100,
- "max": 29500,
- "mean": 22151.35135135135
+ "min": 1333,
+ "max": 12685,
+ "mean": 9525.081081081082
}
}
},
@@ -467,37 +467,37 @@
"day": {
"input": {
"min": 0,
- "max": 13765,
- "mean": 2451.286222222222
+ "max": 19177,
+ "mean": 3461.84
},
"output": {
"min": 0,
- "max": 3900,
- "mean": 728.5333333333333
+ "max": 1677,
+ "mean": 313.26933333333335
}
},
"week": {
"input": {
"min": 0,
- "max": 27593,
- "mean": 17128.552795031057
+ "max": 37719,
+ "mean": 24189.875776397515
},
"output": {
"min": 0,
- "max": 8300,
- "mean": 5090.683229813664
+ "max": 3569,
+ "mean": 2188.9937888198756
}
},
"month": {
"input": {
- "min": 11886,
- "max": 103256,
- "mean": 74532.35135135135
+ "min": 16182,
+ "max": 144179,
+ "mean": 105258.64864864865
},
"output": {
- "min": 3100,
- "max": 29500,
- "mean": 22151.35135135135
+ "min": 1333,
+ "max": 12685,
+ "mean": 9525.081081081082
}
}
},
@@ -505,37 +505,37 @@
"day": {
"input": {
"min": 0,
- "max": 17097,
- "mean": 3239.8124444444443
+ "max": 23618,
+ "mean": 4548.530666666667
},
"output": {
"min": 0,
- "max": 4700,
- "mean": 943.5555555555555
+ "max": 2021,
+ "mean": 405.7288888888889
}
},
"week": {
"input": {
"min": 0,
- "max": 36147,
- "mean": 22638.44099378882
+ "max": 50629,
+ "mean": 31783.211180124224
},
"output": {
"min": 0,
- "max": 11200,
- "mean": 6593.167701863354
+ "max": 4816,
+ "mean": 2835.0621118012423
}
},
"month": {
"input": {
- "min": 13077,
- "max": 122603,
- "mean": 98507.81081081081
+ "min": 17788,
+ "max": 170739,
+ "mean": 138299.9189189189
},
"output": {
- "min": 3400,
- "max": 34700,
- "mean": 28689.18918918919
+ "min": 1462,
+ "max": 14921,
+ "mean": 12336.351351351352
}
}
},
@@ -543,37 +543,37 @@
"day": {
"input": {
"min": 0,
- "max": 23993,
- "mean": 4422.054222222222
+ "max": 33290,
+ "mean": 6221.055111111111
},
"output": {
"min": 0,
- "max": 6700,
- "mean": 1297.1555555555556
+ "max": 2881,
+ "mean": 557.7768888888888
}
},
"week": {
"input": {
"min": 0,
- "max": 48994,
- "mean": 30899.447204968943
+ "max": 70202,
+ "mean": 43470.105590062114
},
"output": {
"min": 0,
- "max": 15300,
- "mean": 9063.975155279502
+ "max": 6579,
+ "mean": 3897.509316770186
}
},
"month": {
"input": {
- "min": 17429,
- "max": 169516,
- "mean": 134454.35135135136
+ "min": 23801,
+ "max": 236368,
+ "mean": 189153.7027027027
},
"output": {
- "min": 4600,
- "max": 48200,
- "mean": 39440.54054054054
+ "min": 1978,
+ "max": 20726,
+ "mean": 16959.432432432433
}
}
}
diff --git a/models/LLM/Tokenizer/token_count.py b/models/LLM/Tokenizer/token_count.py
index 75dd76197..7a620050b 100644
--- a/models/LLM/Tokenizer/token_count.py
+++ b/models/LLM/Tokenizer/token_count.py
@@ -8,7 +8,7 @@ from datetime import datetime, timedelta
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../")))
from testModel.utils import get_article_data
-from variables.pubmed import NCDS_MESH_TERM, KEYWORDS_MESH_TERM, KEYWORDS_MESH_SUBHEADING, KEYWORDS_MESH_SITE_PROPOSITION, KEYWORDS_MESH_PROPOSITION
+from variables.pubmed import NCDS, NCDS_MESH_TERM, KEYWORDS_MESH_TERM, KEYWORDS_MESH_SUBHEADING, KEYWORDS_MESH_SITE_PROPOSITION, KEYWORDS_MESH_PROPOSITION
DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../dataSources/PubMed/data"))
RESULT_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "./data"))
@@ -137,7 +137,14 @@ for tokenizer_name in TOKENIZERS:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
- output_token_num = 100
+ output_exemple = '```json\n'
+ output_exemple += '{'
+ output_exemple += '\'labels\': [' + ', '.join(NCDS) + ']'
+ output_exemple += '}'
+ output_exemple += '```'
+
+ output_tokens = tokenizer(output_exemple, return_tensors="pt")
+ output_token_num = len(output_tokens["input_ids"][0])
i = 1
for article in data:
@@ -147,7 +154,13 @@ for tokenizer_name in TOKENIZERS:
article_date = datetime(int(article["Date"]["Year"]), int(article["Date"]["Month"]), int(article["Date"]["Day"]))
title, abstract = get_article_data(article)
- tokens = tokenizer(title+abstract, return_tensors="pt")
+ prompt = 'I need you to give me the labels that could be given to the text (keep in mind that u can put multiple labels and select only the labels that i give you):\n'
+ prompt += 'text: ' + title + abstract + '\n'
+ prompt += 'labels: ' + "[" + ", ".join(NCDS) + ']\n'
+ prompt += 'Give the response in json format: { "labels": [] }. Follow this writing to the letter and don t add text around it. Only put the labels that you select between [].'
+ prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between "".'
+
+ tokens = tokenizer(prompt, return_tensors="pt")
num_tokens = len(tokens["input_ids"][0])
add_token_number(article_date, start_date, num_tokens, output_token_num, counts, tokenizer_name, "NO KEYWORDS")
diff --git a/models/LLM/prices/data/llm_prices.json b/models/LLM/prices/data/llm_prices.json
index 375cc8a4d..2e2629707 100644
--- a/models/LLM/prices/data/llm_prices.json
+++ b/models/LLM/prices/data/llm_prices.json
@@ -1,338 +1,338 @@
{
"Mistral Large": {
- "ALL": 495.855152,
+ "ALL": 702.3351359999999,
"NO KEYWORDS": {
- "day": 0.032945036912280706,
- "week": 0.23020600326904217,
- "month": 1.0017072034139403
+ "day": 0.03153566877192982,
+ "week": 0.22035793396534814,
+ "month": 0.9588547937411096
},
"KEYWORDS": {
- "day": 0.001468795602339181,
- "week": 0.010263323308270676,
- "month": 0.04465932574679943
+ "day": 0.0014062348538011697,
+ "week": 0.009826175220660347,
+ "month": 0.04275714082503557
},
"SUBHEADINGS": {
- "day": 0.001468795602339181,
- "week": 0.010263323308270676,
- "month": 0.04465932574679943
+ "day": 0.0014062348538011697,
+ "week": 0.009826175220660347,
+ "month": 0.04275714082503557
},
"SITE PROPOSITION": {
- "day": 0.0019225152748538013,
- "week": 0.013433724746649231,
- "month": 0.058454856330014224
+ "day": 0.001841470222222222,
+ "week": 0.012867416149068325,
+ "month": 0.05599064864864864
},
"PROPOSITION": {
- "day": 0.002632865216374269,
- "week": 0.018397350114416476,
- "month": 0.08005333428165007
+ "day": 0.0025214139883040936,
+ "week": 0.01761857600523047,
+ "month": 0.07666461450924608
}
},
"Mistral Small": {
- "ALL": 24.792757599999998,
+ "ALL": 35.116756800000005,
"NO KEYWORDS": {
- "day": 0.0016472518456140351,
- "week": 0.011510300163452107,
- "month": 0.050085360170697014
+ "day": 0.0015767834385964914,
+ "week": 0.011017896698267408,
+ "month": 0.04794273968705548
},
"KEYWORDS": {
- "day": 7.343978011695905e-05,
- "week": 0.0005131661654135338,
- "month": 0.0022329662873399716
+ "day": 7.031174269005849e-05,
+ "week": 0.0004913087610330174,
+ "month": 0.002137857041251779
},
"SUBHEADINGS": {
- "day": 7.343978011695905e-05,
- "week": 0.0005131661654135338,
- "month": 0.0022329662873399716
+ "day": 7.031174269005849e-05,
+ "week": 0.0004913087610330174,
+ "month": 0.002137857041251779
},
"SITE PROPOSITION": {
- "day": 9.612576374269007e-05,
- "week": 0.0006716862373324615,
- "month": 0.002922742816500711
+ "day": 9.20735111111111e-05,
+ "week": 0.0006433708074534163,
+ "month": 0.0027995324324324323
},
"PROPOSITION": {
- "day": 0.00013164326081871344,
- "week": 0.0009198675057208238,
- "month": 0.004002666714082503
+ "day": 0.0001260706994152047,
+ "week": 0.0008809288002615234,
+ "month": 0.003833230725462305
}
},
"GPT-4o": {
- "ALL": 774.773675,
+ "ALL": 1097.39865,
"NO KEYWORDS": {
- "day": 0.04765905052631579,
- "week": 0.3330213157894737,
- "month": 1.4490927524893316
+ "day": 0.04229139040935673,
+ "week": 0.29551437397842434,
+ "month": 1.2858868705547655
},
"KEYWORDS": {
- "day": 0.0021235734502923977,
- "week": 0.014838634357633215,
- "month": 0.06456811166429588
+ "day": 0.0018852869005847954,
+ "week": 0.013173588591042826,
+ "month": 0.05732291251778096
},
"SUBHEADINGS": {
- "day": 0.0021235734502923977,
- "week": 0.014838634357633215,
- "month": 0.06456811166429588
+ "day": 0.0018852869005847954,
+ "week": 0.013173588591042826,
+ "month": 0.05732291251778096
},
"SITE PROPOSITION": {
- "day": 0.002775600233918129,
- "week": 0.019394722131415496,
- "month": 0.08439325035561879
+ "day": 0.00246696,
+ "week": 0.017238074534161492,
+ "month": 0.07500891891891892
},
"PROPOSITION": {
- "day": 0.003803116608187135,
- "week": 0.02657457257273619,
- "month": 0.1156353022759602
+ "day": 0.00337876970760234,
+ "week": 0.023609415658712,
+ "month": 0.1027328627311522
}
},
"GPT-4o mini": {
- "ALL": 46.486420499999994,
+ "ALL": 65.843919,
"NO KEYWORDS": {
- "day": 0.0028595430315789473,
- "week": 0.019981278947368418,
- "month": 0.08694556514935989
+ "day": 0.0025374834245614035,
+ "week": 0.017730862438705457,
+ "month": 0.07715321223328592
},
"KEYWORDS": {
- "day": 0.00012741440701754387,
- "week": 0.0008903180614579927,
- "month": 0.003874086699857752
+ "day": 0.0001131172140350877,
+ "week": 0.0007904153154625694,
+ "month": 0.003439374751066857
},
"SUBHEADINGS": {
- "day": 0.00012741440701754387,
- "week": 0.0008903180614579927,
- "month": 0.003874086699857752
+ "day": 0.0001131172140350877,
+ "week": 0.0007904153154625694,
+ "month": 0.003439374751066857
},
"SITE PROPOSITION": {
- "day": 0.00016653601403508771,
- "week": 0.0011636833278849297,
- "month": 0.005063595021337126
+ "day": 0.00014801759999999998,
+ "week": 0.0010342844720496896,
+ "month": 0.004500535135135135
},
"PROPOSITION": {
- "day": 0.00022818699649122803,
- "week": 0.001594474354364171,
- "month": 0.006938118136557611
+ "day": 0.00020272618245614036,
+ "week": 0.0014165649395227198,
+ "month": 0.006163971763869132
}
},
"DeepSeek-V3": {
- "ALL": 84.91519478000001,
+ "ALL": 120.27489204000001,
"NO KEYWORDS": {
- "day": 0.005198999491929825,
- "week": 0.036328412598888526,
- "month": 0.15807768725462307
+ "day": 0.004590444599766082,
+ "week": 0.03207608804184374,
+ "month": 0.13957432904694167
},
"KEYWORDS": {
- "day": 0.00023164656421052632,
- "week": 0.001618648352402746,
- "month": 0.007043307695590327
+ "day": 0.00020463093192982457,
+ "week": 0.0014298745243543642,
+ "month": 0.006221886443812235
},
"SUBHEADINGS": {
- "day": 0.00023164656421052632,
- "week": 0.001618648352402746,
- "month": 0.007043307695590327
+ "day": 0.00020463093192982457,
+ "week": 0.0014298745243543642,
+ "month": 0.006221886443812235
},
"SITE PROPOSITION": {
- "day": 0.00030274447438596495,
- "week": 0.002115450519777705,
- "month": 0.009205068477951637
+ "day": 0.00026775265777777773,
+ "week": 0.00187094248447205,
+ "month": 0.008141128108108108
},
"PROPOSITION": {
- "day": 0.00041483287438596496,
- "week": 0.002898676917293233,
- "month": 0.01261316172119488
+ "day": 0.00036672314619883045,
+ "week": 0.002562506456358287,
+ "month": 0.011150365931721196
}
},
"Gemini 2.0 Flash": {
- "ALL": 30.990947000000002,
+ "ALL": 43.89594600000001,
"NO KEYWORDS": {
- "day": 0.0019063620210526318,
- "week": 0.013320852631578948,
- "month": 0.05796371009957327
+ "day": 0.0016916556163742691,
+ "week": 0.011820574959136973,
+ "month": 0.051435474822190616
},
"KEYWORDS": {
- "day": 8.49429380116959e-05,
- "week": 0.0005935453743053285,
- "month": 0.002582724466571835
+ "day": 7.541147602339182e-05,
+ "week": 0.000526943543641713,
+ "month": 0.002292916500711238
},
"SUBHEADINGS": {
- "day": 8.49429380116959e-05,
- "week": 0.0005935453743053285,
- "month": 0.002582724466571835
+ "day": 7.541147602339182e-05,
+ "week": 0.000526943543641713,
+ "month": 0.002292916500711238
},
"SITE PROPOSITION": {
- "day": 0.00011102400935672515,
- "week": 0.0007757888852566199,
- "month": 0.003375730014224751
+ "day": 9.86784e-05,
+ "week": 0.0006895229813664598,
+ "month": 0.003000356756756757
},
"PROPOSITION": {
- "day": 0.0001521246643274854,
- "week": 0.0010629829029094476,
- "month": 0.0046254120910384075
+ "day": 0.00013515078830409358,
+ "week": 0.00094437662634848,
+ "month": 0.004109314509246088
}
},
"Gemini 2.0 Flash-Lite": {
- "ALL": 23.243210249999997,
+ "ALL": 32.9219595,
"NO KEYWORDS": {
- "day": 0.0014297715157894737,
- "week": 0.009990639473684209,
- "month": 0.043472782574679944
+ "day": 0.0012687417122807017,
+ "week": 0.008865431219352729,
+ "month": 0.03857660611664296
},
"KEYWORDS": {
- "day": 6.370720350877193e-05,
- "week": 0.00044515903072899637,
- "month": 0.001937043349928876
+ "day": 5.655860701754385e-05,
+ "week": 0.0003952076577312847,
+ "month": 0.0017196873755334285
},
"SUBHEADINGS": {
- "day": 6.370720350877193e-05,
- "week": 0.00044515903072899637,
- "month": 0.001937043349928876
+ "day": 5.655860701754385e-05,
+ "week": 0.0003952076577312847,
+ "month": 0.0017196873755334285
},
"SITE PROPOSITION": {
- "day": 8.326800701754386e-05,
- "week": 0.0005818416639424649,
- "month": 0.002531797510668563
+ "day": 7.400879999999999e-05,
+ "week": 0.0005171422360248448,
+ "month": 0.0022502675675675674
},
"PROPOSITION": {
- "day": 0.00011409349824561402,
- "week": 0.0007972371771820855,
- "month": 0.0034690590682788054
+ "day": 0.00010136309122807018,
+ "week": 0.0007082824697613599,
+ "month": 0.003081985881934566
}
},
"Claude 3.7 Sonnet": {
- "ALL": 1115.674092,
+ "ALL": 1580.2540560000002,
"NO KEYWORDS": {
- "day": 0.06496416589473684,
- "week": 0.4539421529911736,
- "month": 1.975261800853485
+ "day": 0.054195833824561404,
+ "week": 0.37869759660019614,
+ "month": 1.6478462987197726
},
"KEYWORDS": {
- "day": 0.002893382877192982,
- "week": 0.0202177374959137,
- "month": 0.08797447937411096
+ "day": 0.0024153362807017545,
+ "week": 0.01687734978751226,
+ "month": 0.07343927880512092
},
"SUBHEADINGS": {
- "day": 0.002893382877192982,
- "week": 0.0202177374959137,
- "month": 0.08797447937411096
+ "day": 0.0024153362807017545,
+ "week": 0.01687734978751226,
+ "month": 0.07343927880512092
},
"SITE PROPOSITION": {
- "day": 0.0037776676491228073,
- "week": 0.02639674599542334,
- "month": 0.11486151635846373
+ "day": 0.0031584986666666664,
+ "week": 0.022070254658385094,
+ "month": 0.09603543243243243
},
"PROPOSITION": {
- "day": 0.005178182035087719,
- "week": 0.03618294900294214,
- "month": 0.15744472403982931
+ "day": 0.0043269263157894745,
+ "week": 0.030234733573063097,
+ "month": 0.13156194879089617
}
},
"Claude 3.5 Haiku": {
- "ALL": 297.51309119999996,
+ "ALL": 421.4010816,
"NO KEYWORDS": {
- "day": 0.017323777571929825,
- "week": 0.12105124079764629,
- "month": 0.526736480227596
+ "day": 0.014452222353216375,
+ "week": 0.1009860257600523,
+ "month": 0.439425679658606
},
"KEYWORDS": {
- "day": 0.0007715687672514619,
- "week": 0.005391396665576986,
- "month": 0.023459861166429587
+ "day": 0.0006440896748538011,
+ "week": 0.00450062661000327,
+ "month": 0.01958380768136558
},
"SUBHEADINGS": {
- "day": 0.0007715687672514619,
- "week": 0.005391396665576986,
- "month": 0.023459861166429587
+ "day": 0.0006440896748538011,
+ "week": 0.00450062661000327,
+ "month": 0.01958380768136558
},
"SITE PROPOSITION": {
- "day": 0.0010073780397660819,
- "week": 0.007039132265446224,
- "month": 0.030629737695590326
+ "day": 0.000842266311111111,
+ "week": 0.005885401242236026,
+ "month": 0.02560944864864865
},
"PROPOSITION": {
- "day": 0.0013808485426900583,
- "week": 0.00964878640078457,
- "month": 0.04198525974395448
+ "day": 0.0011538470175438598,
+ "week": 0.008062595619483491,
+ "month": 0.03508318634423897
}
},
"Claude 3 Opus": {
- "ALL": 5578.370459999999,
+ "ALL": 7901.27028,
"NO KEYWORDS": {
- "day": 0.3248208294736842,
- "week": 2.2697107649558674,
- "month": 9.876309004267425
+ "day": 0.27097916912280706,
+ "week": 1.8934879830009808,
+ "month": 8.239231493598862
},
"KEYWORDS": {
- "day": 0.01446691438596491,
- "week": 0.10108868747956848,
- "month": 0.4398723968705548
+ "day": 0.012076681403508772,
+ "week": 0.0843867489375613,
+ "month": 0.36719639402560456
},
"SUBHEADINGS": {
- "day": 0.01446691438596491,
- "week": 0.10108868747956848,
- "month": 0.4398723968705548
+ "day": 0.012076681403508772,
+ "week": 0.0843867489375613,
+ "month": 0.36719639402560456
},
"SITE PROPOSITION": {
- "day": 0.018888338245614034,
- "week": 0.1319837299771167,
- "month": 0.5743075817923186
+ "day": 0.01579249333333333,
+ "week": 0.11035127329192547,
+ "month": 0.4801771621621621
},
"PROPOSITION": {
- "day": 0.025890910175438597,
- "week": 0.18091474501471067,
- "month": 0.7872236201991465
+ "day": 0.02163463157894737,
+ "week": 0.15117366786531547,
+ "month": 0.6578097439544808
}
},
"Command A": {
- "ALL": 774.773675,
+ "ALL": 1097.39865,
"NO KEYWORDS": {
- "day": 0.04765905052631579,
- "week": 0.3330213157894737,
- "month": 1.4490927524893316
+ "day": 0.04229139040935673,
+ "week": 0.29551437397842434,
+ "month": 1.2858868705547655
},
"KEYWORDS": {
- "day": 0.0021235734502923977,
- "week": 0.014838634357633215,
- "month": 0.06456811166429588
+ "day": 0.0018852869005847954,
+ "week": 0.013173588591042826,
+ "month": 0.05732291251778096
},
"SUBHEADINGS": {
- "day": 0.0021235734502923977,
- "week": 0.014838634357633215,
- "month": 0.06456811166429588
+ "day": 0.0018852869005847954,
+ "week": 0.013173588591042826,
+ "month": 0.05732291251778096
},
"SITE PROPOSITION": {
- "day": 0.002775600233918129,
- "week": 0.019394722131415496,
- "month": 0.08439325035561879
+ "day": 0.00246696,
+ "week": 0.017238074534161492,
+ "month": 0.07500891891891892
},
"PROPOSITION": {
- "day": 0.003803116608187135,
- "week": 0.02657457257273619,
- "month": 0.1156353022759602
+ "day": 0.00337876970760234,
+ "week": 0.023609415658712,
+ "month": 0.1027328627311522
}
},
"Command R+": {
- "ALL": 774.773675,
+ "ALL": 1097.39865,
"NO KEYWORDS": {
- "day": 0.04765905052631579,
- "week": 0.3330213157894737,
- "month": 1.4490927524893316
+ "day": 0.04229139040935673,
+ "week": 0.29551437397842434,
+ "month": 1.2858868705547655
},
"KEYWORDS": {
- "day": 0.0021235734502923977,
- "week": 0.014838634357633215,
- "month": 0.06456811166429588
+ "day": 0.0018852869005847954,
+ "week": 0.013173588591042826,
+ "month": 0.05732291251778096
},
"SUBHEADINGS": {
- "day": 0.0021235734502923977,
- "week": 0.014838634357633215,
- "month": 0.06456811166429588
+ "day": 0.0018852869005847954,
+ "week": 0.013173588591042826,
+ "month": 0.05732291251778096
},
"SITE PROPOSITION": {
- "day": 0.002775600233918129,
- "week": 0.019394722131415496,
- "month": 0.08439325035561879
+ "day": 0.00246696,
+ "week": 0.017238074534161492,
+ "month": 0.07500891891891892
},
"PROPOSITION": {
- "day": 0.003803116608187135,
- "week": 0.02657457257273619,
- "month": 0.1156353022759602
+ "day": 0.00337876970760234,
+ "week": 0.023609415658712,
+ "month": 0.1027328627311522
}
}
}
\ No newline at end of file
diff --git a/testModel/__pycache__/metrics.cpython-313.pyc b/testModel/__pycache__/metrics.cpython-313.pyc
index aabc6b292b1d019797cd5429382ab3481e71baf9..8e053fa8ce7d94c4bc17d4f237b6f1a6de04508e 100644
GIT binary patch
delta 350
zcmeyuGn=39GcPX}0|NuY^0k-Km$FXelVH@CsIJTv%qqoD#G1~g$-dF=0PEy@R<ZgB
z1_p+y4AG1r)gT<oU<zgDF-9|kL_ioxlqo_;mw|yHk2#b%n9(YNfgzERfgznS7$nZh
zz!1u;%gw-`%gVr@!l23QSH!}=z)-}`z`&r%R3rytOM=)eMLY})47WH7>Wfm7GmA6x
z^KP*fr6wol<P=FTFfbH@bSWq(C=_vnl<}memc*wS#uq2&7o`>{F)%P_ffU&>FfcSQ
z+!2tyAg6a(K)=EB4zI`sQPs=5Y7K4=gk>%Yt2TJwU>BN@c$r=N2EV`!Zk`E}7X_6U
zR9_S@y2x#OfyKCpmyLme!B3NIvLKrjqtaw`Hd!uZ1_lOD2ox(%_G6Rb|HQ+>Xz-bb
MozbL-mw|x+0C<5%hX4Qo
delta 74
zcmbQu|AmL|GcPX}0|NuYt)z?Ti&-Y}NiZr+R99w6XVqle7;=D>*-w*ovL3tCWM6hk
eE_nt921W)3hGMzN1?)0hpLtjq4T^Xf7#IMyv=9gY
diff --git a/testModel/__pycache__/utils.cpython-313.pyc b/testModel/__pycache__/utils.cpython-313.pyc
index 5ef3ac15973dc8324c05ca69bb0aa452f3864f3b..7c4f32213b0aaa36e3175807da59e80a985c85cb 100644
GIT binary patch
delta 1024
zcmX@3)Tzw-nU|M~fq{YH?%K=g&nNOqGP+Du{~PJWTg0ot5X=zF;?3v9U&OD#5X>sY
zP$Uq{CdE)B7|brkP$U%0A;nN69Ly=jP$Uw}CB;xA8q6)lP$U-2BgIf89?UDnP$ZGg
zsLA&dWV~M$v!0&*N|wo$jEa1>c+yi#;?oS{i<9$<Qa8_HRA*!qntX`q1tag|)y&os
zf(#4{w*)|H5{pVQlXFtzQxZ!Oi)1GMWA<khp6thxz$iI+J&TGQSfLn5VR>R+Noq=b
zK~ZW-W^zepeqM2r^5l;!CXB+H)mZ;BGKx=DVYg-!oSeX}#%R2`o!yA3-V9_5Uus2a
zaw$kdd`V_*YLO8G14EH20|UcK)*@q&0=C40g4Dc}A}x@FHi*z+U|`VXxW$o{nUh+O
zSdvi$GN{NLBxL|1z(#0-^m6Csr=;e@r{<LuRTddCFfbHzFfcGEK!Bt|5!m_s#ffF9
z@eq|+#rb(fHk$*vRx$>fg3JON4K@(urXmZF)$B!9FsWOdd8y@4^AUDigQUUMu_LT2
z5}fSKBg<&NIg`heG1eL6JZ489A5ErPe91+ri6yD=MXAN5IVHtKAUA3<fkWaJe@<#%
zdPzona$-qpdVW!5F%#HCh|5Kx8X;~6MNxcNVo{O%WPZMAH(>?_hGbC8!!R2I0|PSy
z1H)%I1_p+y48e@SjQUIv5<-L6!AyFrDh!&;enlXQR<cfB$EU7wi^<KfNFL-VFaa_d
z?0_OykQj)F1BI=|<ez+Unm73PKXSA2uzg@)V`VD>sZC~tIe~$J0Td!&S42&=<#(2k
z2Jyktp$w)_b-|2aH!%4Xc~0);SB4}q-r|z{B19MkFfcHrgDf?ke1>1y;0~YE1!?We
zd^%`WYf2WWgG0KwvbYGGcr-vxwwx>`;GR|lvip`$Sz=LUVp2|Ou^u$BLt>yv2&7U5
zL@0v@P+}~y0kP~sggb};dA$ha6iDcRJPQt(TO2mI`6;D2sdhyXllKXPvUp!+@SZF#
QsKLw5$o5%?oe>f+08xAB(*OVf
delta 939
zcmeBFKBL6@nU|M~fq{WRM&fe%tckpmj1Cjk|As0s1TzG)c=LMk74a!B1hYyp6!8bM
zNih@&1hY#q6bS}%NHG)%1#?O<6bT1&Nih_O1anI<6p04&NHG+N1@lTV6p5!ZYVy4V
z8SPiatf!~Hl6i71qvGZ@jOvVxf|IW>y<p^-yqDQpLV$sR;g&#pYDs)zQAuWUPHKEg
zVo73=^ke}Re@3Cn2`mYW5|a<IsK|j8ih&fCC+3x;ro<N%rKV&imt^MW6&EQ@W@a^E
z6xwXT`j?SWY_b`<HKV}fB6c-Kqs=qfjhO09LALOvR-`7Ef;7aJWag$888R?16sa&U
zFsx)PG6E@JODrfz%}XiL1W9Ot2yF%i22GAz9BG+3sRfB88ATw2ip)S#`XB;qga$}2
zcW!=4YEFD=UP)19kpTk(Loo*f1A_ttNGcS8ozGvKSe6<OQJGbopI2nPIhkuEW1tDh
zEU?jF13_*oG6z}BUStWAy2Y87S`IZIVYd}X8f+aq!nz`X$+0}LjCPx=c{~|aoIuWF
zcJ%SlWV*$dT$Gwvk{Vx>T3nh_Qd|Vmq{&odIQa#yj4arRqEHnOM}p!azAUk*$ZfJB
zU$l!50|P@cC{AIRm4ShQnSp`fvkU_R!&HV~#$ZN$CI|_kgBgRF^jK6FG@1R1m_R|z
zGWignx=N88$lYK9WF|Otid;ZqAR-PFe5#WL`Q_9;a<cNUePCc^Wh(+HOlE}H%)r3F
z2D994as<CKcQ7N^A|}5gkIB3El_3d+x40y~2;pBp1_p*Skm*{JfAcG=qnWNLQKSYA
ztm4YzB5?exgPdeB*-yYd@s?0oVo_#dQch~I9yF;!!mmgWq(~Y>D1iu2QYx|rvFt#E
z8;Ah;st9B`#N!~Zfc<=n!zMRBr8Fnit|)BsZ-G!2+lvgglS2eGc-a`)J`1ohLc9$C
DcP-IR
diff --git a/testModel/launch_tests.py b/testModel/launch_tests.py
new file mode 100644
index 000000000..19c6c51d9
--- /dev/null
+++ b/testModel/launch_tests.py
@@ -0,0 +1,25 @@
+import subprocess
+import sys
+import os
+import argparse
+
+# Ajouter le répertoire parent au chemin de recherche
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../")))
+
+from variables.models import MODELS
+
+TEST_SCRIPT_FILEPATH = "./test.py"
+
+file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), TEST_SCRIPT_FILEPATH))
+
+
+
+args = ["-m"]
+
+try:
+ result = subprocess.run(["python", file_path], check=True, capture_output=True, text=True)
+ print("Script output:", result.stdout)
+except subprocess.CalledProcessError as e:
+ print("Script crashed with error:", e.stderr)
+except FileNotFoundError:
+ print("Python executable or script not found.")
\ No newline at end of file
--
GitLab