YAML Updates

projet/resources/playbooks/swiss-impex-petroleum.yml

 clients: # The default management port for a client is 18965
-  - "127.0.0.1"
+  - "129.194.187.141"
+  - "129.194.187.142"
+  - "129.194.187.143"
+  - "129.194.187.144"
+  - "129.194.187.145"
+  - "129.194.187.146"
+  - "129.194.187.147"
+  - "129.194.187.148"
+  - "129.194.187.149"
+  - "129.194.187.150"
+  - "129.194.187.151"
+  - "129.194.187.152"
+  - "129.194.187.153"
+  - "129.194.187.154"
+  - "129.194.187.155"
+  - "129.194.187.156"
+  - "129.194.187.157"
+  - "129.194.187.158"
+  - "129.194.187.127"
+  - "129.194.187.128"
+  - "129.194.187.129"
+  - "129.194.187.130"
 
 variables:
   - merchandiseId: "2709.0090"

projet/src/client/spiders/seleniumspiders/SwissImpex/Parser.py

@@ -129,25 +129,30 @@ class Parser:
 		self.w()
 
 		# Selecting year
+		WebDriverWait(self.driver, 5).until(EC.visibility_of_element_located((By.XPATH, DATE_SELECTION_YEAR_START)))
 		yStart = Select(self.driver.find_element_by_xpath(DATE_SELECTION_YEAR_START))
 		yStart.select_by_visible_text(str(self.selectedYear))
 		self.w()
 
+		WebDriverWait(self.driver, 5).until(EC.visibility_of_element_located((By.XPATH, DATE_SELECTION_YEAR_END)))
 		yEnd = Select(self.driver.find_element_by_xpath(DATE_SELECTION_YEAR_END))
 		yEnd.select_by_visible_text(str(self.selectedYear))
 		self.w()
 
 		# Selecting month
+		WebDriverWait(self.driver, 5).until(EC.visibility_of_element_located((By.XPATH, DATE_SELECTION_MONTH_START)))
 		mStart = Select(self.driver.find_element_by_xpath(DATE_SELECTION_MONTH_START))
 		mStart.select_by_visible_text(str(self.selectedMonth))
 		self.w()
 
+		WebDriverWait(self.driver, 5).until(EC.visibility_of_element_located((By.XPATH, DATE_SELECTION_MONTH_END)))
 		mEnd = Select(self.driver.find_element_by_xpath(DATE_SELECTION_MONTH_END))
 		mEnd.select_by_visible_text(str(self.selectedMonth))
 		logger.info(
 			"Selected {0} {1}...Sending manifesto to server.".format(self.selectedMonth, self.selectedYear))
 		self.w()
 
+		WebDriverWait(self.driver, 5).until(EC.visibility_of_element_located((By.XPATH, DATE_SELECTION_SEND_MANIFESTO)))
 		sendManifestToServer = self.driver.find_element_by_xpath(DATE_SELECTION_SEND_MANIFESTO)
 		sendManifestToServer.click()
 		self.w()

projet/src/common/utils/PerfObserver.py

@@ -2,6 +2,7 @@ import time
 from threading import Thread
 
 from loguru import logger
+from sqlalchemy import func
 from sqlalchemy.orm import Session
 
 from common.mvc import getSession
@@ -13,13 +14,11 @@ from common.networking.intercom import Intercom
 
 
 class PerformanceObserver:
-	db: Session = None  # Central database
 	shouldStop: bool = False  # Should stop monitoring
 	clients: list = []  # Clients list
 	timing: int = 1  # Timing between every measure
 
 	def __init__(self, clients, timing):
-		self.db = getSession()
 		self.folder = env("FILE_LOCKER")
 		self.clients = clients
 		self.timing = timing
@@ -39,12 +38,15 @@ class PerformanceObserver:
 		            .format(shouldMonitorTasksInDB, shouldMonitorFilesInFileLocker))
 
 		secondsPassed = 0
+		lastFileCount = 0
 		with open('measures.csv', 'a') as fd:
 			while not self.shouldStop:
 				currentRun = [str(secondsPassed)]
 
 				if shouldMonitorTasksInDB:
-					currentRun.append(str(self.db.query(Task).count()))
+					db = getSession()
+					currentRun.append(str(db.query(func.count(Task.id)).scalar()))
+					db.close()
 
 				if shouldMonitorFilesInFileLocker:
 					total = 0
@@ -52,11 +54,14 @@ class PerformanceObserver:
 					for client in self.clients:
 						intercom = Intercom(client, env("CLIENT_PORT"), True)
 						intercom.talk({"type": "statrep"})
+						statrep = {}
 						try:
-							statrep = intercom.listen(500)
+							statrep = intercom.listen(750)
 						except:
-							statrep = {"files": []}
+							total = lastFileCount  # Safe
+							break
 						total += len(statrep.get("files") or [])
+						lastFileCount = total
 					currentRun.append(str(total))
 
 				fd.write(",".join(currentRun) + "\n")
@@ -66,5 +71,4 @@ class PerformanceObserver:
 
 	def stop(self):
 		self.shouldStop = True
-		self.db.close()
 		logger.info("Stopped observer with success.")

projet/src/common/utils/YAMLParser.py

@@ -37,7 +37,7 @@ class YAMLParser:
 			try:
 				intercom = Intercom(client, int(env("CLIENT_PORT")), True)
 				intercom.talk({"type": "heartbeat", "payload": ""})
-				message = intercom.listen(300)
+				message = intercom.listen(750)
 				if "type" not in message:
 					logger.error("Received corrupted message from client {0}.".format(client))
 					raise RuntimeError

projet/src/common/utils/starters.py

@@ -4,14 +4,12 @@ from collections import OrderedDict
 from os import getenv as env
 
 from loguru import logger
-from scrapy.crawler import CrawlerProcess, CrawlerRunner
 
-from client.spiders.scrapy.Spiders.spiders.faogeneve_spider import FAOGeneveSpider
 from client.spiders.seleniumspiders.SwissImpex.Parser import Parser
 from common.utils.FileParser import FileParser
 from common.utils.RegexParser import RegexParser
 from common.utils.XLSXParser import XLSXParser
-from twisted.internet import reactor
+
 
 def startFAOGeneve(pages):
 	logger.warning("Starting scrapy. Since this is a third party, logs will not be kept here.")