Skip to content
Snippets Groups Projects
Commit 2106a87c authored by Ivan Pavlovich's avatar Ivan Pavlovich
Browse files

Recalculated tokens and LLM price with real input/output prompts

parent e5e43b0d
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 447 additions and 409 deletions
models/LLM/Ollama/ollama_wrapper.py
+ 1
− 1
View file @ 2106a87c
...@@ -19,7 +19,7 @@ def classify(model, sequence, labels): ...@@ -19,7 +19,7 @@ def classify(model, sequence, labels):
prompt += 'text: ' + sequence + '\n' prompt += 'text: ' + sequence + '\n'
prompt += 'labels: ' + "[" + ", ".join(labels) + ']\n' prompt += 'labels: ' + "[" + ", ".join(labels) + ']\n'
prompt += 'Give the response in json format: { "labels": [] }. Follow this writing to the letter and don t add text around it. Only put the labels that you select between [].' prompt += 'Give the response in json format: { "labels": [] }. Follow this writing to the letter and don t add text around it. Only put the labels that you select between [].'
prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between \'\'.' prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between "".'
response: ChatResponse = chat(model=model, messages=[ response: ChatResponse = chat(model=model, messages=[
{ {
......
models/LLM/Tokenizer/data/token_count.json
+ 213
− 213
View file @ 2106a87c
This diff is collapsed.
models/LLM/Tokenizer/token_count.py
+ 16
− 3
View file @ 2106a87c
...@@ -8,7 +8,7 @@ from datetime import datetime, timedelta ...@@ -8,7 +8,7 @@ from datetime import datetime, timedelta
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../")))
from testModel.utils import get_article_data from testModel.utils import get_article_data
from variables.pubmed import NCDS_MESH_TERM, KEYWORDS_MESH_TERM, KEYWORDS_MESH_SUBHEADING, KEYWORDS_MESH_SITE_PROPOSITION, KEYWORDS_MESH_PROPOSITION from variables.pubmed import NCDS, NCDS_MESH_TERM, KEYWORDS_MESH_TERM, KEYWORDS_MESH_SUBHEADING, KEYWORDS_MESH_SITE_PROPOSITION, KEYWORDS_MESH_PROPOSITION
DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../dataSources/PubMed/data")) DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../dataSources/PubMed/data"))
RESULT_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "./data")) RESULT_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "./data"))
...@@ -137,7 +137,14 @@ for tokenizer_name in TOKENIZERS: ...@@ -137,7 +137,14 @@ for tokenizer_name in TOKENIZERS:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name) tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
output_token_num = 100 output_exemple = '```json\n'
output_exemple += '{'
output_exemple += '\'labels\': [' + ', '.join(NCDS) + ']'
output_exemple += '}'
output_exemple += '```'
output_tokens = tokenizer(output_exemple, return_tensors="pt")
output_token_num = len(output_tokens["input_ids"][0])
i = 1 i = 1
for article in data: for article in data:
...@@ -147,7 +154,13 @@ for tokenizer_name in TOKENIZERS: ...@@ -147,7 +154,13 @@ for tokenizer_name in TOKENIZERS:
article_date = datetime(int(article["Date"]["Year"]), int(article["Date"]["Month"]), int(article["Date"]["Day"])) article_date = datetime(int(article["Date"]["Year"]), int(article["Date"]["Month"]), int(article["Date"]["Day"]))
title, abstract = get_article_data(article) title, abstract = get_article_data(article)
tokens = tokenizer(title+abstract, return_tensors="pt") prompt = 'I need you to give me the labels that could be given to the text (keep in mind that u can put multiple labels and select only the labels that i give you):\n'
prompt += 'text: ' + title + abstract + '\n'
prompt += 'labels: ' + "[" + ", ".join(NCDS) + ']\n'
prompt += 'Give the response in json format: { "labels": [] }. Follow this writing to the letter and don t add text around it. Only put the labels that you select between [].'
prompt += 'Even if you have no labels give me a response like: { "labels": [] }. And also put labels between "".'
tokens = tokenizer(prompt, return_tensors="pt")
num_tokens = len(tokens["input_ids"][0]) num_tokens = len(tokens["input_ids"][0])
add_token_number(article_date, start_date, num_tokens, output_token_num, counts, tokenizer_name, "NO KEYWORDS") add_token_number(article_date, start_date, num_tokens, output_token_num, counts, tokenizer_name, "NO KEYWORDS")
......
models/LLM/prices/data/llm_prices.json
+ 192
− 192
View file @ 2106a87c
{ {
"Mistral Large": { "Mistral Large": {
"ALL": 495.855152, "ALL": 702.3351359999999,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.032945036912280706, "day": 0.03153566877192982,
"week": 0.23020600326904217, "week": 0.22035793396534814,
"month": 1.0017072034139403 "month": 0.9588547937411096
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.001468795602339181, "day": 0.0014062348538011697,
"week": 0.010263323308270676, "week": 0.009826175220660347,
"month": 0.04465932574679943 "month": 0.04275714082503557
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.001468795602339181, "day": 0.0014062348538011697,
"week": 0.010263323308270676, "week": 0.009826175220660347,
"month": 0.04465932574679943 "month": 0.04275714082503557
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.0019225152748538013, "day": 0.001841470222222222,
"week": 0.013433724746649231, "week": 0.012867416149068325,
"month": 0.058454856330014224 "month": 0.05599064864864864
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.002632865216374269, "day": 0.0025214139883040936,
"week": 0.018397350114416476, "week": 0.01761857600523047,
"month": 0.08005333428165007 "month": 0.07666461450924608
} }
}, },
"Mistral Small": { "Mistral Small": {
"ALL": 24.792757599999998, "ALL": 35.116756800000005,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.0016472518456140351, "day": 0.0015767834385964914,
"week": 0.011510300163452107, "week": 0.011017896698267408,
"month": 0.050085360170697014 "month": 0.04794273968705548
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 7.343978011695905e-05, "day": 7.031174269005849e-05,
"week": 0.0005131661654135338, "week": 0.0004913087610330174,
"month": 0.0022329662873399716 "month": 0.002137857041251779
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 7.343978011695905e-05, "day": 7.031174269005849e-05,
"week": 0.0005131661654135338, "week": 0.0004913087610330174,
"month": 0.0022329662873399716 "month": 0.002137857041251779
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 9.612576374269007e-05, "day": 9.20735111111111e-05,
"week": 0.0006716862373324615, "week": 0.0006433708074534163,
"month": 0.002922742816500711 "month": 0.0027995324324324323
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.00013164326081871344, "day": 0.0001260706994152047,
"week": 0.0009198675057208238, "week": 0.0008809288002615234,
"month": 0.004002666714082503 "month": 0.003833230725462305
} }
}, },
"GPT-4o": { "GPT-4o": {
"ALL": 774.773675, "ALL": 1097.39865,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.04765905052631579, "day": 0.04229139040935673,
"week": 0.3330213157894737, "week": 0.29551437397842434,
"month": 1.4490927524893316 "month": 1.2858868705547655
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.0021235734502923977, "day": 0.0018852869005847954,
"week": 0.014838634357633215, "week": 0.013173588591042826,
"month": 0.06456811166429588 "month": 0.05732291251778096
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.0021235734502923977, "day": 0.0018852869005847954,
"week": 0.014838634357633215, "week": 0.013173588591042826,
"month": 0.06456811166429588 "month": 0.05732291251778096
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.002775600233918129, "day": 0.00246696,
"week": 0.019394722131415496, "week": 0.017238074534161492,
"month": 0.08439325035561879 "month": 0.07500891891891892
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.003803116608187135, "day": 0.00337876970760234,
"week": 0.02657457257273619, "week": 0.023609415658712,
"month": 0.1156353022759602 "month": 0.1027328627311522
} }
}, },
"GPT-4o mini": { "GPT-4o mini": {
"ALL": 46.486420499999994, "ALL": 65.843919,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.0028595430315789473, "day": 0.0025374834245614035,
"week": 0.019981278947368418, "week": 0.017730862438705457,
"month": 0.08694556514935989 "month": 0.07715321223328592
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.00012741440701754387, "day": 0.0001131172140350877,
"week": 0.0008903180614579927, "week": 0.0007904153154625694,
"month": 0.003874086699857752 "month": 0.003439374751066857
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.00012741440701754387, "day": 0.0001131172140350877,
"week": 0.0008903180614579927, "week": 0.0007904153154625694,
"month": 0.003874086699857752 "month": 0.003439374751066857
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.00016653601403508771, "day": 0.00014801759999999998,
"week": 0.0011636833278849297, "week": 0.0010342844720496896,
"month": 0.005063595021337126 "month": 0.004500535135135135
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.00022818699649122803, "day": 0.00020272618245614036,
"week": 0.001594474354364171, "week": 0.0014165649395227198,
"month": 0.006938118136557611 "month": 0.006163971763869132
} }
}, },
"DeepSeek-V3": { "DeepSeek-V3": {
"ALL": 84.91519478000001, "ALL": 120.27489204000001,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.005198999491929825, "day": 0.004590444599766082,
"week": 0.036328412598888526, "week": 0.03207608804184374,
"month": 0.15807768725462307 "month": 0.13957432904694167
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.00023164656421052632, "day": 0.00020463093192982457,
"week": 0.001618648352402746, "week": 0.0014298745243543642,
"month": 0.007043307695590327 "month": 0.006221886443812235
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.00023164656421052632, "day": 0.00020463093192982457,
"week": 0.001618648352402746, "week": 0.0014298745243543642,
"month": 0.007043307695590327 "month": 0.006221886443812235
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.00030274447438596495, "day": 0.00026775265777777773,
"week": 0.002115450519777705, "week": 0.00187094248447205,
"month": 0.009205068477951637 "month": 0.008141128108108108
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.00041483287438596496, "day": 0.00036672314619883045,
"week": 0.002898676917293233, "week": 0.002562506456358287,
"month": 0.01261316172119488 "month": 0.011150365931721196
} }
}, },
"Gemini 2.0 Flash": { "Gemini 2.0 Flash": {
"ALL": 30.990947000000002, "ALL": 43.89594600000001,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.0019063620210526318, "day": 0.0016916556163742691,
"week": 0.013320852631578948, "week": 0.011820574959136973,
"month": 0.05796371009957327 "month": 0.051435474822190616
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 8.49429380116959e-05, "day": 7.541147602339182e-05,
"week": 0.0005935453743053285, "week": 0.000526943543641713,
"month": 0.002582724466571835 "month": 0.002292916500711238
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 8.49429380116959e-05, "day": 7.541147602339182e-05,
"week": 0.0005935453743053285, "week": 0.000526943543641713,
"month": 0.002582724466571835 "month": 0.002292916500711238
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.00011102400935672515, "day": 9.86784e-05,
"week": 0.0007757888852566199, "week": 0.0006895229813664598,
"month": 0.003375730014224751 "month": 0.003000356756756757
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.0001521246643274854, "day": 0.00013515078830409358,
"week": 0.0010629829029094476, "week": 0.00094437662634848,
"month": 0.0046254120910384075 "month": 0.004109314509246088
} }
}, },
"Gemini 2.0 Flash-Lite": { "Gemini 2.0 Flash-Lite": {
"ALL": 23.243210249999997, "ALL": 32.9219595,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.0014297715157894737, "day": 0.0012687417122807017,
"week": 0.009990639473684209, "week": 0.008865431219352729,
"month": 0.043472782574679944 "month": 0.03857660611664296
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 6.370720350877193e-05, "day": 5.655860701754385e-05,
"week": 0.00044515903072899637, "week": 0.0003952076577312847,
"month": 0.001937043349928876 "month": 0.0017196873755334285
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 6.370720350877193e-05, "day": 5.655860701754385e-05,
"week": 0.00044515903072899637, "week": 0.0003952076577312847,
"month": 0.001937043349928876 "month": 0.0017196873755334285
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 8.326800701754386e-05, "day": 7.400879999999999e-05,
"week": 0.0005818416639424649, "week": 0.0005171422360248448,
"month": 0.002531797510668563 "month": 0.0022502675675675674
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.00011409349824561402, "day": 0.00010136309122807018,
"week": 0.0007972371771820855, "week": 0.0007082824697613599,
"month": 0.0034690590682788054 "month": 0.003081985881934566
} }
}, },
"Claude 3.7 Sonnet": { "Claude 3.7 Sonnet": {
"ALL": 1115.674092, "ALL": 1580.2540560000002,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.06496416589473684, "day": 0.054195833824561404,
"week": 0.4539421529911736, "week": 0.37869759660019614,
"month": 1.975261800853485 "month": 1.6478462987197726
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.002893382877192982, "day": 0.0024153362807017545,
"week": 0.0202177374959137, "week": 0.01687734978751226,
"month": 0.08797447937411096 "month": 0.07343927880512092
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.002893382877192982, "day": 0.0024153362807017545,
"week": 0.0202177374959137, "week": 0.01687734978751226,
"month": 0.08797447937411096 "month": 0.07343927880512092
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.0037776676491228073, "day": 0.0031584986666666664,
"week": 0.02639674599542334, "week": 0.022070254658385094,
"month": 0.11486151635846373 "month": 0.09603543243243243
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.005178182035087719, "day": 0.0043269263157894745,
"week": 0.03618294900294214, "week": 0.030234733573063097,
"month": 0.15744472403982931 "month": 0.13156194879089617
} }
}, },
"Claude 3.5 Haiku": { "Claude 3.5 Haiku": {
"ALL": 297.51309119999996, "ALL": 421.4010816,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.017323777571929825, "day": 0.014452222353216375,
"week": 0.12105124079764629, "week": 0.1009860257600523,
"month": 0.526736480227596 "month": 0.439425679658606
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.0007715687672514619, "day": 0.0006440896748538011,
"week": 0.005391396665576986, "week": 0.00450062661000327,
"month": 0.023459861166429587 "month": 0.01958380768136558
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.0007715687672514619, "day": 0.0006440896748538011,
"week": 0.005391396665576986, "week": 0.00450062661000327,
"month": 0.023459861166429587 "month": 0.01958380768136558
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.0010073780397660819, "day": 0.000842266311111111,
"week": 0.007039132265446224, "week": 0.005885401242236026,
"month": 0.030629737695590326 "month": 0.02560944864864865
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.0013808485426900583, "day": 0.0011538470175438598,
"week": 0.00964878640078457, "week": 0.008062595619483491,
"month": 0.04198525974395448 "month": 0.03508318634423897
} }
}, },
"Claude 3 Opus": { "Claude 3 Opus": {
"ALL": 5578.370459999999, "ALL": 7901.27028,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.3248208294736842, "day": 0.27097916912280706,
"week": 2.2697107649558674, "week": 1.8934879830009808,
"month": 9.876309004267425 "month": 8.239231493598862
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.01446691438596491, "day": 0.012076681403508772,
"week": 0.10108868747956848, "week": 0.0843867489375613,
"month": 0.4398723968705548 "month": 0.36719639402560456
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.01446691438596491, "day": 0.012076681403508772,
"week": 0.10108868747956848, "week": 0.0843867489375613,
"month": 0.4398723968705548 "month": 0.36719639402560456
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.018888338245614034, "day": 0.01579249333333333,
"week": 0.1319837299771167, "week": 0.11035127329192547,
"month": 0.5743075817923186 "month": 0.4801771621621621
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.025890910175438597, "day": 0.02163463157894737,
"week": 0.18091474501471067, "week": 0.15117366786531547,
"month": 0.7872236201991465 "month": 0.6578097439544808
} }
}, },
"Command A": { "Command A": {
"ALL": 774.773675, "ALL": 1097.39865,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.04765905052631579, "day": 0.04229139040935673,
"week": 0.3330213157894737, "week": 0.29551437397842434,
"month": 1.4490927524893316 "month": 1.2858868705547655
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.0021235734502923977, "day": 0.0018852869005847954,
"week": 0.014838634357633215, "week": 0.013173588591042826,
"month": 0.06456811166429588 "month": 0.05732291251778096
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.0021235734502923977, "day": 0.0018852869005847954,
"week": 0.014838634357633215, "week": 0.013173588591042826,
"month": 0.06456811166429588 "month": 0.05732291251778096
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.002775600233918129, "day": 0.00246696,
"week": 0.019394722131415496, "week": 0.017238074534161492,
"month": 0.08439325035561879 "month": 0.07500891891891892
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.003803116608187135, "day": 0.00337876970760234,
"week": 0.02657457257273619, "week": 0.023609415658712,
"month": 0.1156353022759602 "month": 0.1027328627311522
} }
}, },
"Command R+": { "Command R+": {
"ALL": 774.773675, "ALL": 1097.39865,
"NO KEYWORDS": { "NO KEYWORDS": {
"day": 0.04765905052631579, "day": 0.04229139040935673,
"week": 0.3330213157894737, "week": 0.29551437397842434,
"month": 1.4490927524893316 "month": 1.2858868705547655
}, },
"KEYWORDS": { "KEYWORDS": {
"day": 0.0021235734502923977, "day": 0.0018852869005847954,
"week": 0.014838634357633215, "week": 0.013173588591042826,
"month": 0.06456811166429588 "month": 0.05732291251778096
}, },
"SUBHEADINGS": { "SUBHEADINGS": {
"day": 0.0021235734502923977, "day": 0.0018852869005847954,
"week": 0.014838634357633215, "week": 0.013173588591042826,
"month": 0.06456811166429588 "month": 0.05732291251778096
}, },
"SITE PROPOSITION": { "SITE PROPOSITION": {
"day": 0.002775600233918129, "day": 0.00246696,
"week": 0.019394722131415496, "week": 0.017238074534161492,
"month": 0.08439325035561879 "month": 0.07500891891891892
}, },
"PROPOSITION": { "PROPOSITION": {
"day": 0.003803116608187135, "day": 0.00337876970760234,
"week": 0.02657457257273619, "week": 0.023609415658712,
"month": 0.1156353022759602 "month": 0.1027328627311522
} }
} }
} }
\ No newline at end of file
testModel/__pycache__/metrics.cpython-313.pyc
+ 0
− 0
View file @ 2106a87c
No preview for this file type
testModel/__pycache__/utils.cpython-313.pyc
+ 0
− 0
View file @ 2106a87c
No preview for this file type
testModel/launch_tests.py 0 → 100644
+ 25
− 0
View file @ 2106a87c
import subprocess
import sys
import os
import argparse
# Ajouter le répertoire parent au chemin de recherche
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../")))
from variables.models import MODELS
TEST_SCRIPT_FILEPATH = "./test.py"
file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), TEST_SCRIPT_FILEPATH))
args = ["-m"]
try:
result = subprocess.run(["python", file_path], check=True, capture_output=True, text=True)
print("Script output:", result.stdout)
except subprocess.CalledProcessError as e:
print("Script crashed with error:", e.stderr)
except FileNotFoundError:
print("Python executable or script not found.")
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment