diff --git a/projet/agent/src/client/client.py b/projet/agent/src/client/client.py
index 629f180f131dfe33306250bc6753884c6001a1a7..a128a5d3d037e086aecb75589f270361748c0569 100644
--- a/projet/agent/src/client/client.py
+++ b/projet/agent/src/client/client.py
@@ -78,7 +78,7 @@ class Client:
elif message["type"] == "throttle":
logger.critical("Received throttle request from server. Stopping all activities for 1 minute.")
self.pool.pause()
- threading.Timer(60, self.pool.resume).start()
+ threading.Timer(180, self.pool.resume).start()
intercom.ok()
# UNKNOWN MESSAGE
diff --git a/projet/agent/src/common/schnell/workers.py b/projet/agent/src/common/schnell/workers.py
index 89c7ec86d724d339a21f4e07db152198aa6978ae..d3ce0dfb58774ab106d739535d44941bfffb9651 100644
--- a/projet/agent/src/common/schnell/workers.py
+++ b/projet/agent/src/common/schnell/workers.py
@@ -6,6 +6,11 @@ from os import getenv as env
from common.tasks.Runner import Runner
+def setup(event):
+ global unpaused
+ unpaused = event
+
+
def work(data):
context = {}
@@ -33,16 +38,26 @@ def work(data):
class Schnell:
nWorkers: int = 0
pool: multiprocessing.Pool = None
+ chief: multiprocessing.Event = None
def __init__(self, nWorkers: int):
self.nWorkers = nWorkers
+ self.chief = multiprocessing.Event()
def start(self):
logger.info("Starting pool with {0} workers...".format(self.nWorkers))
- self.pool = multiprocessing.get_context("spawn").Pool(processes=self.nWorkers)
+ self.pool = multiprocessing.get_context("spawn").Pool(self.nWorkers, setup, (self.chief, ))
+ self.chief.set()
# Stops the schnell pool.
def stop(self):
logger.warning("Stopping schnell pool...")
self.pool.terminate()
logger.warning("Done.")
+
+ def pause(self):
+ self.chief.clear()
+
+ def resume(self):
+ self.chief.set()
+ logger.success("Unfrozen.")
diff --git a/projet/agent/src/server/api/ThrottleAPI.py b/projet/agent/src/server/api/ThrottleAPI.py
new file mode 100644
index 0000000000000000000000000000000000000000..4210d02ba05253cba86004e59de03790b93d5b54
--- /dev/null
+++ b/projet/agent/src/server/api/ThrottleAPI.py
@@ -0,0 +1,14 @@
+import falcon
+
+from loguru import logger
+from os import getenv as env
+from common.networking.intercom import Intercom
+
+
+class ThrottleAPI(object):
+ def on_get(self, req: falcon.Request, resp: falcon.Response):
+ if "client" in req.params:
+ clientIntercom = Intercom(req.params['client'], int(env("CLIENT_PORT")), True)
+ clientIntercom.talk({"type": "throttle"})
+ clientIntercom.listen()
+ logger.info("Sent throttle to client because of many errors.")
diff --git a/projet/agent/src/server/api/__init__.py b/projet/agent/src/server/api/__init__.py
index 386c87c12779b989da1cf6c395a4dd262703ff6a..5626ad0040405b6ec95b25871a252a688b088e53 100644
--- a/projet/agent/src/server/api/__init__.py
+++ b/projet/agent/src/server/api/__init__.py
@@ -6,6 +6,7 @@ from common.schnell.workers import Schnell
from server.api.DocumentationAPI import DocumentationAPI
from server.api.FileAPI import FileAPI
from server.api.PlaybookAPI import PlaybookAPI, ArgumentAPI
+from server.api.ThrottleAPI import ThrottleAPI
class API(gunicorn.app.base.BaseApplication):
@@ -40,4 +41,6 @@ def start(schnell: Schnell):
app.add_route('/playbook', api)
app.add_route('/playbookArgs', ArgumentAPI())
+ app.add_route("/throttle", ThrottleAPI())
+
API(app, options).run()
diff --git a/projet/agent/src/start.py b/projet/agent/src/start.py
index 3b3d0037b78c232fedf0c4a93cb309e156b5f9af..efc6b3519407cbc3153b6b3f88e15b7096ba7368 100644
--- a/projet/agent/src/start.py
+++ b/projet/agent/src/start.py
@@ -7,11 +7,9 @@ from loguru import logger
from client.client import Client
from common.schnell.workers import Schnell
-from server.io.YAMLParser import YAMLParser
from os import getenv as env
from server import api
-from server.io.shotgun import Shotgun
# Environment awareness activation
load_dotenv(find_dotenv())
@@ -23,56 +21,18 @@ group.add_argument("--server", help="Considers this instance as a server", defau
group.add_argument("--client", help="Considers this instance as a client. On by default.", default=True,
action='store_true')
-parser.add_argument("--dry", help="Does not actually send any task anywhere. Just shows the jobs produced by the YAML "
- "file.", default=False, action='store_true')
parser.add_argument("--remote", help="Allows remote jobs to be sent to this instance. Only used when working in "
"client mode.", default=False, action='store_true')
-parser.add_argument("--perf-files", help="Monitor processed files in file locker for performance analysis",
- default=False, action='store_true')
-parser.add_argument("--perf-db", help="Monitor tasks in central database for performance analysis",
- default=False, action='store_true')
-parser.add_argument("--perf-timing", help="Timing between each performance analysis measurement. Default : 1",
- default=1)
args = parser.parse_args()
# Identity awareness evaluation
isServer = args.server
-isDry = args.dry
allowRemote = args.remote
-shouldMonitorFiles = args.perf_files
-shouldMonitorDB = args.perf_db
-shouldMonitor = shouldMonitorFiles or shouldMonitorDB
-monitorTiming = args.perf_timing
-
-
-# Executes a playbook. SERVER ONLY
-def runPlaybook(p: YAMLParser, clts: list):
- c, t = p.evaluateJobs(isDry)
- logger.info("Created {0} jobs.".format(len(t)))
-
- if isDry:
- logger.warning("DRY MODE - No job will be dispatched or stored into the database, but the logs will tell you "
- "how things will play out.")
- logger.info("Jobs found : \n{0}".format("\n".join(t)))
-
- shotgun = Shotgun(clts)
- r = shotgun.divideJobs(t)
- logger.info("Dispatching {0} jobs to {1} clients.".format(len(t), len(r)))
- if isDry:
- logger.info("Job repartition : \n{0}".format(r))
-
- return c, t, r
# Starting program
if __name__ == '__main__':
- if not isServer and shouldMonitor:
- logger.error("You cannot monitor performance with a Inari client.")
- exit(300)
-
- logger.debug("Connecting to DB...")
-
logger.info("Inari - Théo Pirkl")
if not allowRemote:
logger.warning("You have forbidden outside instances to connect to this instance. \nThat's fine, but you will "
@@ -99,7 +59,7 @@ if __name__ == '__main__':
else:
logger.add("run_client.log", format="{time} \t| {process.id} \t| {level} | {message}", enqueue=True)
# Specific imports
- from client.spiders.seleniumspiders import HTTPHandler
+ from client import HTTPHandler
logger.info("Running in client mode. All order received by any server will be processed.")
clientPool = Schnell(int(env("CLIENT_WORKERS")))
diff --git a/projet/frontend/app/Console/Commands/TriggerClientCheck.php b/projet/frontend/app/Console/Commands/TriggerClientCheck.php
new file mode 100644
index 0000000000000000000000000000000000000000..96147e3c8d1c535839f59a91118e2b6dafbf12a8
--- /dev/null
+++ b/projet/frontend/app/Console/Commands/TriggerClientCheck.php
@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\ThrottleFaultClients;
+use Illuminate\Console\Command;
+
+class TriggerClientCheck extends Command
+{
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'client:check';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Checks the clients against potential crashes';
+
+ /**
+ * Create a new command instance.
+ *
+ * @return void
+ */
+ public function __construct()
+ {
+ parent::__construct();
+ }
+
+ /**
+ * Execute the console command.
+ *
+ * @return mixed
+ */
+ public function handle() {
+ ThrottleFaultClients::dispatch();
+ }
+}
diff --git a/projet/frontend/app/Console/Kernel.php b/projet/frontend/app/Console/Kernel.php
index 56997261b12f5ef259d0de32e72c399b5f2fb949..6fe1041fa2646f2df208117db8451a23e97b6d26 100644
--- a/projet/frontend/app/Console/Kernel.php
+++ b/projet/frontend/app/Console/Kernel.php
@@ -3,6 +3,7 @@
namespace App\Console;
use App\Console\Commands\CleanOldJobs;
+use App\Console\Commands\TriggerClientCheck;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@@ -26,6 +27,7 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule)
{
$schedule->command(CleanOldJobs::class)->everyTenMinutes();
+ $schedule->command(TriggerClientCheck::class)->everyMinute();
}
/**
diff --git a/projet/frontend/app/Http/Controllers/API/JobController.php b/projet/frontend/app/Http/Controllers/API/JobController.php
index 65437f9ca883faeaa9385fba97dbdbf583bce7c4..10e88849337389e97aaaec53f2ea1f963cf7867b 100644
--- a/projet/frontend/app/Http/Controllers/API/JobController.php
+++ b/projet/frontend/app/Http/Controllers/API/JobController.php
@@ -8,6 +8,7 @@ use App\Models\Client;
use App\Models\Job;
use Illuminate\Database\Eloquent\ModelNotFoundException;
use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Http;
class JobController extends Controller {
diff --git a/projet/frontend/app/Http/Controllers/ProgressController.php b/projet/frontend/app/Http/Controllers/ProgressController.php
new file mode 100644
index 0000000000000000000000000000000000000000..29716e11f1c4586028dcd16de137d04604a7fb31
--- /dev/null
+++ b/projet/frontend/app/Http/Controllers/ProgressController.php
@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use App\Models\Job;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
+
+class ProgressController extends Controller {
+ public function index(){
+ $data = DB::select('SELECT status, COUNT(status) as cnt FROM jobs GROUP BY status');
+ $count = Job::where('updated_at', '>=', now()->subMinute())->whereStatus(2)->count();
+ if (collect($data)->isNotEmpty()){
+
+ if (collect($data)->count() != 3){
+ return abort(418, "Nous sommes en attente des workers. Revenez plus tard.");
+ }
+ $data[0]->status = "En attente";
+ $data[1]->status = "échoués";
+ $data[2]->status = "terminés";
+ } else {
+ $data = false;
+ }
+
+
+ return view('pages.progress.index', ['data' => $data, 'count' => $count]);
+ }
+}
diff --git a/projet/frontend/app/Jobs/ThrottleFaultClients.php b/projet/frontend/app/Jobs/ThrottleFaultClients.php
new file mode 100644
index 0000000000000000000000000000000000000000..07bd242108f0ff308ae3c2c8d657d32ec58be38c
--- /dev/null
+++ b/projet/frontend/app/Jobs/ThrottleFaultClients.php
@@ -0,0 +1,44 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Client;
+use App\Models\Job;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Http;
+
+class ThrottleFaultClients implements ShouldQueue
+{
+ use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+ /**
+ * Create a new job instance.
+ *
+ * @return void
+ */
+ public function __construct() {
+ //
+ }
+
+ /**
+ * Execute the job.
+ *
+ * @return void
+ */
+ public function handle() {
+ foreach (Client::all() as $client){
+ $clientJobsFailedCount = $client->jobs()
+ ->where('created_at', '>=', now()->subMinutes(5))
+ ->whereStatus(1)
+ ->count();
+
+ if ($clientJobsFailedCount >= 10){
+ Http::get("http://127.0.0.1:8080/throttle", ['client' => $client->id]);
+ }
+ }
+ }
+}
diff --git a/projet/frontend/public/img/wait.jpeg b/projet/frontend/public/img/wait.jpeg
new file mode 100644
index 0000000000000000000000000000000000000000..c63ead1697135ce272da70e8b1533fd788500686
Binary files /dev/null and b/projet/frontend/public/img/wait.jpeg differ
diff --git a/projet/frontend/resources/views/errors/418.blade.php b/projet/frontend/resources/views/errors/418.blade.php
new file mode 100644
index 0000000000000000000000000000000000000000..f9353ea6f902e6dc9b50041dee4e1909066c239f
--- /dev/null
+++ b/projet/frontend/resources/views/errors/418.blade.php
@@ -0,0 +1,11 @@
+@extends('errors::illustrated-layout')
+
+@section('code', '418')
+@section('title', "Une minute...")
+
+@section('image')
+<div style="background-image: url({{ asset('/svg/403.svg') }});" class="absolute pin bg-cover bg-no-repeat md:bg-left lg:bg-center">
+</div>
+@endsection
+
+@section('message', $exception->getMessage())
diff --git a/projet/frontend/resources/views/pages/progress/index.blade.php b/projet/frontend/resources/views/pages/progress/index.blade.php
new file mode 100644
index 0000000000000000000000000000000000000000..db3152d7d4dc99ad8434555e36c2293e7f030583
--- /dev/null
+++ b/projet/frontend/resources/views/pages/progress/index.blade.php
@@ -0,0 +1,23 @@
+@extends('errors.illustrated-layout')
+
+@section('code', 'Progrès actuel')
+
+@section('image')
+ <div style="background-image: url({{ asset('/img/wait.jpeg') }});" class="absolute pin bg-cover bg-no-repeat md:bg-left lg:bg-center">
+ </div>
+@endsection
+
+@section('message')
+ @if($data)
+ Actuellement, le système traite les documents de cette façon :
+ <ul>
+ <li>{{ number_format($data[0]->cnt, 0, ',', ' ') }} documents {{ strtolower($data[0]->status) }}</li>
+ <li>{{ number_format($data[1]->cnt, 0, ',', ' ') }} documents {{ strtolower($data[1]->status) }}</li>
+ <li>{{ number_format($data[2]->cnt, 0, ',', ' ') }} documents {{ strtolower($data[2]->status) }}</li>
+ </ul>
+ Soit {{ $count }} documents traités sans erreur par minute.
+ @else
+ Aucun document en traitement.
+ @endif
+ <div class="w-16 h-1 bg-indigo-light my-3 md:my-6"></div>
+@endsection
diff --git a/projet/frontend/routes/web.php b/projet/frontend/routes/web.php
index 8997529edc280ab531319c97f1104094b0982434..3e708dc36c9e5f1dd883800ed6953c3c90812d79 100644
--- a/projet/frontend/routes/web.php
+++ b/projet/frontend/routes/web.php
@@ -38,3 +38,5 @@ Route::get("/client/{client}/resume", "JobController@resume")->name('job.resume'
Route::get('/client', 'ClientController@index')->name('client');
Route::get('/click', 'ClickController@index')->name('click');
+
+Route::get('/progress', 'ProgressController@index');