Modification du model de test

api/pubmedApi.py

 from requests import get
 from parser.xmlParser import parseXmlFile
 import model.facebookBartLargeMnli as facebookBartLargeMnli
+import json
 
 FILENAME = "pubmedData.xml"
 
 # term = "diabetes+type+1+OR+diabetes+type+2+OR+mental+health"
-term = '"Diabetes+Mellitus"[Mesh]'
+#term = '"Diabetes+Mellitus"[Mesh:noexp]'
+#term = '"Diabetes+Mellitus%2C+Type+1"[Mesh:noexp]'
+term = '"Diabetes+Mellitus%2C+Type+2"[Mesh:noexp]'
 # term = 'cancer'
-date_min = "2024/11/29"
-date_max = "2024/11/29"
+date_min = "2024/12/01"
+date_max = "2024/12/15"
 
 print(f"Term: {term}")
 print(f"Date min: {date_min}")
@@ -41,6 +44,8 @@ print(obj["PubmedArticle"][0].get("Title"))
 
 # Some articles have no abstract: 39611868, 39608382, 39608962, 39608966
 
+json_list = []
+
 for key in obj.keys():
     if isinstance(obj[key], list):
 
@@ -53,30 +58,38 @@ for key in obj.keys():
 
                 if "MeshHeadingList" in entrie["MedlineCitation"]:
                     i+=1
+
+                    data = {}
+
                     print(f"Index: {obj[key].index(entrie)}")
-                    pmid = entrie["MedlineCitation"]["PMID"]
-                    title = entrie["MedlineCitation"]["Article"]["Journal"]["Title"]
-                    articleTitle = entrie["MedlineCitation"]["Article"]["ArticleTitle"]
-                    print(f"PMID: {pmid}")
-                    print(f"Title: {title}")
-                    print(f"ArticleTitle: {articleTitle}")
+                    data["PMID"] = entrie["MedlineCitation"]["PMID"]
+                    data["Title"] = entrie["MedlineCitation"]["Article"]["Journal"]["Title"]
+                    data["ArticleTitle"] = entrie["MedlineCitation"]["Article"]["ArticleTitle"]
+                    print(f"PMID: {data["PMID"]}")
+                    print(f"Title: {data["Title"]}")
+                    print(f"ArticleTitle: {data["ArticleTitle"]}")
 
                     if "Abstract" in entrie["MedlineCitation"]["Article"] :
-                        abstract = entrie["MedlineCitation"]["Article"]["Abstract"]["AbstractText"]
-                        print(f"Abstract: {abstract}")
+                        data["Abstract"] = entrie["MedlineCitation"]["Article"]["Abstract"]["AbstractText"]
+                        print(f"Abstract: {data["Abstract"]}")
+
+                    data["Predictions"] = []
 
                     meshTerms = []
                     for meshTerm in entrie["MedlineCitation"]["MeshHeadingList"]["MeshHeading"]:
                         meshTerms.append(meshTerm["DescriptorName"])
 
+                    data["MeshTerms"] = meshTerms
+
                     print(meshTerms)
 
-                    input("Press Enter to continue...")
+                    json_list.append(data)
+
+                    # input("Press Enter to continue...")
 
                     if i > 20:
                         break
 
-
             # print(f"Index: {obj[key].index(entrie)}")
             # pmid = entrie["MedlineCitation"]["PMID"]
             # title = entrie["MedlineCitation"]["Article"]["Journal"]["Title"]
@@ -118,3 +131,6 @@ for key in obj.keys():
         print(f"{key} has multiple entries")
     else:
         print(f"{key} has 1 entry")
+
+with open("../model/data/diabetes_type_2.json", "w") as json_file:
+    json.dump(json_list, json_file, indent=4)
\ No newline at end of file

api/test_model.py

-from parser.jsonParser import parseJsonFile
-import time
-import model.facebookBartLargeMnli as facebookBartLargeMnli
-
-def predict(article):
-    pmid = article["PMID"]
-    title = article["Title"]
-    articleTitle = article["ArticleTitle"]
-    abstract = article["Abstract"]
-    meshTerms = article["MeshTerms"]
-    print(f"PMID: {pmid}")
-    print(f"Title: {title}")
-    print(f"ArticleTitle: {articleTitle}")
-    print(f"Abstract: {abstract}")
-    print(f"MeshTerm: {meshTerms}")
-
-    return facebookBartLargeMnli.classify(title + " \n " + articleTitle + " \n " + abstract)
-
-
-
-try:
-    articles = parseJsonFile("./model/test.json")
-    # print(articles)
-except Exception as e:
-    print(f"Error: {e}")
-
-start = time.time()
-
-predictions = 0
-
-for article in articles:
-    results = predict(article)
-
-    print(f"Labels: {results["labels"]}")
-    print(f"Scores: {results["scores"]}")
-
-    predictPresent = False
-
-    for id, score in enumerate(results["scores"]):
-        if score >= 0.6:
-            if results["labels"][id] in article["Predictions"]:
-                predictPresent = True
-                break
-
-    print(f"Prediction: {predictPresent}")
-
-    if predictPresent and len(article["Predictions"]) != 0:
-        predictions += 1
-    elif predictPresent == False and len(article["Predictions"]) == 0:
-        predictions += 1
-
-end = time.time()
-
-print(f"Time to classify all articles: {end-start} seconds")
-print(f"Good classification: {predictions}/{len(articles)}")
\ No newline at end of file

model/test_model.py

+import sys
+import os
+
+# Ajouter le répertoire parent au chemin de recherche
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../")))
+
+from api.parser.jsonParser import parseJsonFile
+import time
+import api.model.facebookBartLargeMnli as facebookBartLargeMnli
+import statistics
+
+TRESHOLD = 0.6
+
+KEYWORDS = [
+        "Diabetes",
+        "Diabetes type 1",
+        "Diabetes type 2"
+        ]
+
+def predict(article):
+    pmid = article["PMID"]
+    title = article["Title"]
+    articleTitle = article["ArticleTitle"]
+    abstract = article["Abstract"]
+    pred = article["Predictions"]
+    meshTerms = article["MeshTerms"]
+    print(f"PMID: {pmid}")
+    print(f"Title: {title}")
+    print(f"ArticleTitle: {articleTitle}")
+    print(f"Abstract: {abstract}")
+    print(f"Predictions: {pred}")
+    print(f"MeshTerm: {meshTerms}")
+
+    return facebookBartLargeMnli.classify(title + " \n " + articleTitle + " \n " + abstract)
+
+start = time.time()
+
+predictions = []
+selected_scores = []
+
+for key in KEYWORDS:
+
+    try:
+        articles = parseJsonFile(f"./data/{key.replace(" ", "_").lower()}.json")
+        # print(articles)
+    except Exception as e:
+        print(f"Error: {e}")
+
+    for article in articles:
+        results = predict(article)
+
+        print(f"Labels: {results["labels"]}")
+        print(f"Scores: {results["scores"]}")
+
+        selected_labels = []
+
+        for id, score in enumerate(results["scores"]):
+            if score >= TRESHOLD:
+                selected_labels.append(results["labels"][id])
+                selected_scores.append(score)
+
+        for keyword in article["Predictions"]:
+            predictions.append( True if keyword not in selected_labels else False)
+
+end = time.time()
+
+good_predictions_count = predictions.count(True)
+mean_score = statistics.mean(selected_scores)
+median_score = statistics.median(selected_scores)
+
+print(f"Time to classify all articles: {end-start} seconds")
+print(f"Good classification: {good_predictions_count}/{len(predictions)}")
+print(f"Mean score: {mean_score}")
+print(f"Median score: {median_score}")
+print()
\ No newline at end of file