Save audio board and get correct audio on switch shape works

.env

-# Frontend variables.
+# Frontend variables
 BOARDS_FRONTEND_PORT=4200
 ELECTRON='false'
 

backend/src/index.php

@@ -44,6 +44,14 @@ try {
 
           }
           break;
+          case 'audios':
+            if ($key) {
+              $preparedStatement = $db->prepare('SELECT * FROM `audios` WHERE idBoard = :idBoard');
+              $preparedStatement->bindValue('idBoard', $key, PDO::PARAM_INT);
+              $preparedStatement->execute();
+              $rep = $preparedStatement->fetchAll(PDO::FETCH_ASSOC);
+            }
+            break;
       }
       break;
 
@@ -60,6 +68,7 @@ try {
             'hash'    => filter_var($receivedData['hash'],    FILTER_SANITIZE_STRING),
             'title'   => filter_var($receivedData['title'],     FILTER_SANITIZE_STRING),
             'creatorId' => filter_var($receivedData['creatorId'],   FILTER_VALIDATE_INT),
+            'currentShapeID' => filter_var($receivedData['currentShapeID'],   FILTER_VALIDATE_INT),
           );
 
           if (empty($receivedData['graphics'])) {
@@ -72,9 +81,9 @@ try {
 
           $sql = <<<'END'
             INSERT INTO boards
-            (`uid`, `creatorId`, `title`, `height`, `width`, `hash`, `graphics`, `creationDate`)
+            (`uid`, `creatorId`, `title`, `height`, `width`, `hash`, `graphics`, `creationDate`, `currentShapeID`)
             VALUES
-            (NULL, :creatorId, :title, :height, :width, :hash, :graphics, NOW());
+            (NULL, :creatorId, :title, :height, :width, :hash, :graphics, NOW(), :currentShapeID);
             END;
 
           try {
@@ -85,6 +94,7 @@ try {
             $preparedStatement->bindValue('title',    $validationData['title'],    PDO::PARAM_STR);
             $preparedStatement->bindValue('width',    $validationData['width'],    PDO::PARAM_INT);
             $preparedStatement->bindValue('hash',     $validationData['hash'],     PDO::PARAM_STR);
+            $preparedStatement->bindValue('currentShapeID',     $validationData['currentShapeID'],     PDO::PARAM_STR);
 
 
             // graphics on the map
@@ -110,54 +120,38 @@ try {
           }
           break;
 
-          case 'audio':
+          case 'audios':
             /*
              * insert a map in the DB and get back the id
              */
             $receivedData = json_decode(file_get_contents('php://input'), true);
+
             $validationData = array(
               'idBoard'  => filter_var($receivedData['idBoard'],    FILTER_VALIDATE_INT),
               'idShape'   => filter_var($receivedData['idShape'],     FILTER_VALIDATE_INT),
-              'hash'    => filter_var($receivedData['hash'],    FILTER_SANITIZE_STRING),
-              'audio'   => filter_var($receivedData['audio'],     FILTER_SANITIZE_STRING),
+              'audioEncoded'   => filter_var($receivedData['audioEncoded'],     FILTER_SANITIZE_STRING),
+              'audioName'   => filter_var($receivedData['audioName'],     FILTER_SANITIZE_STRING),
             );
 
-            if (empty($receivedData['graphics'])) {
-              $validationData['graphics'] = null;
-            } else {
-              $validationData['graphics'] = json_encode($receivedData['graphics']);
-            }
 
             $rep = $validationData;
 
             $sql = <<<'END'
-              INSERT INTO boards
-              (`uid`, `creatorId`, `title`, `height`, `width`, `hash`, `graphics`, `creationDate`)
+              INSERT INTO audios
+              (`idBoard`, `idShape`, `audioName`, `audioEncoded`, `creationDate`)
               VALUES
-              (NULL, :creatorId, :title, :height, :width, :hash, :graphics, NOW());
+              (:idBoard, :idShape, :audioName, :audioEncoded, NOW());
               END;
 
             try {
               $preparedStatement = $db->prepare($sql);
 
-              $preparedStatement->bindValue('creatorId',  $validationData['creatorId'],  PDO::PARAM_INT);
-              $preparedStatement->bindValue('height',   $validationData['height'],   PDO::PARAM_INT);
-              $preparedStatement->bindValue('title',    $validationData['title'],    PDO::PARAM_STR);
-              $preparedStatement->bindValue('width',    $validationData['width'],    PDO::PARAM_INT);
-              $preparedStatement->bindValue('hash',     $validationData['hash'],     PDO::PARAM_STR);
-
-
-              // graphics on the map
-              if (is_null($validationData['graphics'])) {
-                $preparedStatement->bindValue('graphics', null, PDO::PARAM_NULL);
-              } else {
-                $preparedStatement->bindValue('graphics', $validationData['graphics'], PDO::PARAM_STR);
-              }
-
+              $preparedStatement->bindValue('idBoard',  $validationData['idBoard'],  PDO::PARAM_INT);
+              $preparedStatement->bindValue('idShape',  $validationData['idShape'],   PDO::PARAM_INT);
+              $preparedStatement->bindValue('audioName',$validationData['audioName'],    PDO::PARAM_STR);
+              $preparedStatement->bindValue('audioEncoded',$validationData['audioEncoded'],    PDO::PARAM_STR);
 
               $preparedStatement->execute();
-              $id= $db->lastInsertId();
-              $rep = array('id' => $id);
 
             } catch (Exception $e) {
               if ($conf['displayError']) {

boards-database/maps.sql

@@ -68,6 +68,8 @@ COMMIT;
 CREATE TABLE `audios` (
   `idBoard` INT NOT NULL ,
   `idShape` INT NOT NULL ,
-  `audioBlob` MEDIUMBLOB NOT NULL ,
+  `audioName` varchar(255) DEFAULT NULL,
+  `audioEncoded` longtext DEFAULT NULL,
+  `creationDate` datetime NOT NULL,
   PRIMARY KEY (`idBoard`,`idShape`)
-) ENGINE = InnoDB;
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

frontend/node_modules/@fortawesome/free-solid-svg-icons/package.json

 {
-  "_from": "@fortawesome/free-solid-svg-icons",
+  "_from": "@fortawesome/free-solid-svg-icons@^5.14.0",
   "_id": "@fortawesome/free-solid-svg-icons@5.14.0",
   "_inBundle": false,
   "_integrity": "sha512-M933RDM8cecaKMWDSk3FRYdnzWGW7kBBlGNGfvqLVwcwhUPNj9gcw+xZMrqBdRqxnSXdl3zWzTCNNGEtFUq67Q==",
   "_location": "/@fortawesome/free-solid-svg-icons",
   "_phantomChildren": {},
   "_requested": {
-    "type": "tag",
+    "type": "range",
     "registry": true,
-    "raw": "@fortawesome/free-solid-svg-icons",
+    "raw": "@fortawesome/free-solid-svg-icons@^5.14.0",
     "name": "@fortawesome/free-solid-svg-icons",
     "escapedName": "@fortawesome%2ffree-solid-svg-icons",
     "scope": "@fortawesome",
-    "rawSpec": "",
+    "rawSpec": "^5.14.0",
     "saveSpec": null,
-    "fetchSpec": "latest"
+    "fetchSpec": "^5.14.0"
   },
   "_requiredBy": [
     "#USER",
@@ -22,7 +22,7 @@
   ],
   "_resolved": "https://registry.npmjs.org/@fortawesome/free-solid-svg-icons/-/free-solid-svg-icons-5.14.0.tgz",
   "_shasum": "970453f5e8c4915ad57856c3a0252ac63f6fec18",
-  "_spec": "@fortawesome/free-solid-svg-icons",
+  "_spec": "@fortawesome/free-solid-svg-icons@^5.14.0",
   "_where": "/home/reddevil/Documents/abaplans/boards/frontend",
   "author": {
     "name": "Dave Gandy",

frontend/src/app/nav/nav.component.css

@@ -36,8 +36,16 @@
   margin: 4px 10px 0px;
 }
 
+.btns {
+  border: 1px solid white;
+}
+
 .btn {
   border: 1px solid white;
+  margin: 7px 0px 10px 3px;
+  border-style: solid;
+  border-width: 1px;
+  border-radius: 5px;
 }
 
 .toolbar {

frontend/src/app/nav/nav.component.html

@@ -25,9 +25,24 @@
   <!-- Drawing tools buttons -->
   <div>
     <div class="buttons-group">
+      <div class="buttons-group">
+
+        <div class= btn-group>
+          <label class="btn btn-secondary" for="files">Choisir un arrière plan</label>
+          <input id="files" type="file" style="display:none;" (change)="handle($event.target.files)" accept=".jpeg,.png,.jpg">
+        </div>
+        <button
+          class="btns btn-secondary toolbar-button"
+          tooltip="Modif arrière plan"
+          placement="bottom"
+          (click)='editBackImg()'>
+        </button>
+      </div>
+
+
     <div class="buttons-group">
       <button
-        class="btn btn-secondary toolbar-button"
+        class="btns btn-secondary toolbar-button"
         [ngClass]="{selected: selectedButton['ellipse']===true}"
         tooltip="Ellipse"
         placement="bottom"
@@ -35,7 +50,7 @@
         <!-- <fa-icon [icon]="faCircle"></fa-icon> -->
       </button>
       <button
-        class="btn btn-secondary toolbar-button"
+        class="btns btn-secondary toolbar-button"
         [ngClass]="{selected: selectedButton['rectangle']===true}"
         tooltip="Rectangle"
         placement="bottom"
@@ -43,7 +58,7 @@
         <!-- <fa-icon [icon]="faCircle"></fa-icon> -->
       </button>
       <button
-        class="btn btn-secondary toolbar-button"
+        class="btns btn-secondary toolbar-button"
         [ngClass]="{selected: selectedButton['line']===true}"
         tooltip="Ligne"
         placement="bottom"
@@ -51,7 +66,7 @@
         <!-- <fa-icon [icon]="faCircle"></fa-icon> -->
       </button>
       <button
-        class="btn btn-secondary toolbar-button"
+        class="btns btn-secondary toolbar-button"
         [ngClass]="{selected: selectedButton['polygon']===true}"
         tooltip="Polygone"
         placement="bottom"
@@ -59,7 +74,7 @@
         <!-- <fa-icon [icon]="faCircle"></fa-icon> -->
       </button>
       <button
-        class="btn btn-secondary toolbar-button"
+        class="btns btn-secondary toolbar-button"
         [ngClass]="{selected: selectedButton['pencil']===true}"
         tooltip="Crayon"
         placement="bottom"
@@ -70,19 +85,13 @@
 
     <div class="buttons-group">
       <button
-        class="btn btn-secondary toolbar-button"
-        tooltip="Modif arrière plan"
-        placement="bottom"
-        (click)='changeBackImg()'>
-      </button>
-      <button
-        class="btn btn-secondary toolbar-button"
+        class="btns btn-secondary toolbar-button"
         tooltip="Sauvegarder"
         placement="bottom"
         (click)='save()'>
       </button>
       <button
-        class="btn btn-secondary toolbar-button"
+        class="btns btn-secondary toolbar-button"
         tooltip="Imprimer"
         placement="bottom"
         (click)='print()'>

frontend/src/app/nav/nav.component.ts

@@ -11,14 +11,14 @@ export class NavComponent implements OnInit {
   @Output() setShape = new EventEmitter<String>();
   @Output() saveBoard = new EventEmitter<any>();
   @Output() printBoard = new EventEmitter<any>();
-  @Output() changeBackgroundImg = new EventEmitter<any>();
+  @Output() editBackgroundImg = new EventEmitter<any>();
+  @Output() changeBackgroundImg = new EventEmitter<string>();
 
   constructor(private shapeService: ShapeService) { }
 
   ngOnInit(): void {
     console.log("init");
   }
-
   select(param: string){
     this.setShape.emit(param);
   }
@@ -31,7 +31,14 @@ export class NavComponent implements OnInit {
     this.printBoard.emit();
   }
 
-  changeBackImg(){
-    this.changeBackgroundImg.emit();
+  editBackImg(){
+    this.editBackgroundImg.emit();
+  }
+
+  handle(file : FileList){
+    console.log(file);
+    const url = URL.createObjectURL(file[0]);
+    console.log(url);
+    this.changeBackgroundImg.emit(url);
   }
 }

frontend/src/app/recorder/recorder.component.html

@@ -2,7 +2,8 @@
   <button (click)="initiateRecording()" class="btn btn-primary" *ngIf="!recording"> Start Recording </button>
   <button (click)="stopRecording()" class="btn btn-danger" *ngIf="recording"> Stop Recording </button>
   <div class="clearfix"></div>
-  <audio controls="" *ngIf="selectedShape && shapeAudio[selectedShape.getAttr('id')]">
+
+  <audio id=audio controls="" *ngIf="selectedShape && shapeAudio[selectedShape.getAttr('id')]">
       <source [src]="sanitize(shapeAudio[selectedShape.getAttr('id')])" type="audio/wav">
   </audio>
 </div>

frontend/src/app/recorder/recorder.component.ts

@@ -23,9 +23,23 @@ export class RecorderComponent {
    constructor(private domSanitizer: DomSanitizer) {
    }
    sanitize(url:string){
-     console.log(url)
        return this.domSanitizer.bypassSecurityTrustUrl(url);
    }
+
+   ngOnChanges(): void {
+     this.loadAudioInPlayer()
+  }
+  loadAudioInPlayer(){
+    if(this.selectedShape && this.shapeAudio[this.selectedShape.getAttr('id')]){
+      let audio = document.getElementById('audio') as HTMLMediaElement
+      if(audio){
+        console.log(audio)
+        audio.src = this.shapeAudio[this.selectedShape.getAttr('id')]
+        audio.load()
+      }
+   }
+  }
+
    /**
     * Start recording.
     */
@@ -66,14 +80,8 @@ export class RecorderComponent {
     */
    async processRecording(blob) {
        this.url = URL.createObjectURL(blob);
-       console.log(this.shapeAudio[this.selectedShape.getAttr('id')]);
-
-       console.log(blob);
-       let blobFromURL = await fetch(this.url).then(r => r.blob());
-       console.log(blobFromURL)
-
        this.shapeAudio[this.selectedShape.getAttr('id')]=this.url;
-       console.log(this.shapeAudio);
+       this.loadAudioInPlayer();
    }
    /**
     * Process Error.
@@ -82,4 +90,5 @@ export class RecorderComponent {
        this.error = 'Can not play audio in your browser';
    }
 
+
 }

frontend/src/app/whiteboard-page/boards-database/boards-database.service.ts

 import { Injectable } from '@angular/core';
 import { HttpClient, HttpHeaders } from '@angular/common/http';
 import { environment } from '../../../environments/environment';
-import { Board, Drawing } from './utils';
+import { Board, Audio } from './utils';
 
 @Injectable({
   providedIn: 'root'
@@ -31,6 +31,15 @@ export class BoardsDatabaseService {
     return this.http.get<Board>(`${environment.boardServerUrl}/boards/${uid}`).toPromise();
   }
 
+  /**
+   * Get the list of all existing maps on the maps server.
+   *
+   * @returns A Promise containing the list of maps saved on the maps server.
+   */
+  async getAudios(idBoard :string): Promise<Audio[]> {
+    return this.http.get<Audio[]>(`${environment.boardServerUrl}/audios/${idBoard}`).toPromise();
+  }
+
   /**
    * Save a new map on the maps server.
    *
@@ -54,7 +63,30 @@ export class BoardsDatabaseService {
         title: board.title,
         height: board.height,
         width: board.width,
-        graphics: board.graphics
+        graphics: board.graphics,
+        currentShapeID: board.currentShapeID
+      },
+      httpOptions
+    ).toPromise();
+  }
+
+  async saveAudio(audio : Audio): Promise<{}> {
+    // Set the http header to avoid sending an OPTIONS request before the POST,
+    // which would cause CORS issues.
+    console.log(`${environment.boardServerUrl}/audio/`)
+    const httpOptions = {
+      headers: new HttpHeaders({
+        'Content-Type':  'text/plain',
+      })
+    };
+
+    return this.http.post<{id: number}>(
+    `${environment.boardServerUrl}/audios/`,
+      {
+        idBoard: audio.idBoard,
+        idShape: audio.idShape,
+        audioName : audio.audioName,
+        audioEncoded : audio.audioEncoded
       },
       httpOptions
     ).toPromise();

frontend/src/app/whiteboard-page/boards-database/utils.ts

@@ -8,13 +8,12 @@ export interface Board {
   height?: number;
   width?: number;
   graphics?: string;
-  currentShapeID :number,
+  currentShapeID :number;
 }
 
-export interface Drawing {
-  uid?: number;
-  title: string;
-  height?: number;
-  width?: number;
-  graphics?: string;
+export interface Audio {
+  idBoard : string;
+  idShape : string;
+  audioName : string;
+  audioEncoded : string;
 }

frontend/src/app/whiteboard-page/whiteboard-page.component.html

-<app-nav (setShape)="setSelection($event); addShape($event)" [selectedButton]="selectedButton" (saveBoard)="saveBoard()" (printBoard)="printBoard()" (changeBackgroundImg)="activeChangeOnBackground()"></app-nav>
+<app-nav (setShape)="setSelection($event); addShape($event)" [selectedButton]="selectedButton" (saveBoard)="saveBoard()" (printBoard)="printBoard()" (changeBackgroundImg)="addBackground($event)" (editBackgroundImg)="editBackground()"></app-nav>
 <table>
   <tr>
     <td id=aside width=100%>

frontend/src/app/whiteboard-page/whiteboard-page.component.ts

@@ -5,7 +5,7 @@ import { ShapeService } from '../shape/shape.service';
 import { PrinterService } from '../printer/printer.service';
 import { BoardsDatabaseService } from './boards-database/boards-database.service';
 import { NavComponent } from '../nav/nav.component';
-import { Board } from './boards-database/utils';
+import { Board, Audio } from './boards-database/utils';
 import { MainMenuComponent } from './main-menu/main-menu.component';
 import { ModalModule } from 'ngx-bootstrap/modal';
 import { BoardsMenuComponent } from './boards-menu/boards-menu.component';
@@ -23,6 +23,7 @@ export class WhiteboardPageComponent implements OnInit {
   @ViewChild('boardsMenu') private boardsMenuModal: BoardsMenuComponent;
   @ViewChild('availableSpace') private availableSpace: ElementRef<HTMLDivElement>;
   @ViewChild('container') private container: ElementRef<HTMLDivElement>;
+  @ViewChild('audioPlayer') audioPlayerRef: ElementRef<HTMLAudioElement>;
 
   width: number = window.innerWidth-330;
   height: number = window.innerHeight-104;
@@ -46,7 +47,7 @@ export class WhiteboardPageComponent implements OnInit {
   private openedBoard: Board;
   boardId = '';
   boardTitle = '';
-  backgroundImg: Konva.Image;
+  backgroundImg: Konva.Image = null;
 
 
   constructor(
@@ -77,8 +78,6 @@ export class WhiteboardPageComponent implements OnInit {
     this.addDeleteListener();
 
     //this.addBackground();
-
-
   }
 
   ngAfterViewInit(): void {
@@ -93,8 +92,10 @@ export class WhiteboardPageComponent implements OnInit {
       this.boardsMenuModal.open = false;
       const board = await this.boardsDbService.getBoard(id);
       this.openedBoard = board;
+      this.boardId = '' + board.uid;
+      this.boardTitle = board.title;
       this.currentShapeID = board.currentShapeID;
-      console.log(this.openedBoard.graphics)
+      await this.loadAudios();
       this.stage = Konva.Node.create(this.openedBoard.graphics, 'container');
       this.layer = (this.stage.getLayers().toArray()[0] as Konva.Layer);
       this.initTextures();
@@ -111,6 +112,29 @@ export class WhiteboardPageComponent implements OnInit {
       this.mainMenuModal.open = true;
     };
   }
+
+
+  b64toBlob(dataURI) {
+
+    var byteString = atob(dataURI.split(',')[1]);
+    var ab = new ArrayBuffer(byteString.length);
+    var ia = new Uint8Array(ab);
+
+    for (var i = 0; i < byteString.length; i++) {
+        ia[i] = byteString.charCodeAt(i);
+    }
+    return new Blob([ab], { type: 'audio/wav' });
+  }
+
+  private async loadAudios(){
+    var audios : Audio[] = await this.boardsDbService.getAudios(this.boardId);
+    for(let audio of audios){
+      let blob = this.b64toBlob(audio.audioEncoded);
+      let blobUrl = URL.createObjectURL(blob);
+      this.shapeAudio[audio.idShape]=blobUrl;
+    }
+  }
+
   private resizeBoardView() {
     // it from influencing the availableSpace container's size.
     let availableSpaceWidth = window.innerWidth-330;
@@ -185,15 +209,18 @@ export class WhiteboardPageComponent implements OnInit {
 
   }
 
-  addBackground(){
+  addBackground(fileURL : string){
     var component = this
     var imageObj = new Image();
-
-    imageObj.src = '/assets/swiss.jpg';
+    if(component.backgroundImg != null){
+      component.backgroundImg.remove();
+    }
+    imageObj.src = fileURL;
     imageObj.onload = function() {
       var background = new Konva.Image({
           x: 0,
           y: 0,
+          zIndex: 0,
           image: imageObj,
           draggable:false,
           listening : false,
@@ -201,13 +228,16 @@ export class WhiteboardPageComponent implements OnInit {
       // add the image to the layer
       component.layer.add(background);
       component.backgroundImg = background;
+      component.backgroundImg.moveToBottom();
       component.layer.batchDraw();
     };
 
   }
-  activeChangeOnBackground(){
+  editBackground(){
     this.backgroundImg.listening(!this.backgroundImg.listening());
     this.backgroundImg.draggable(!this.backgroundImg.draggable());
+    this.layer.batchDraw()
+
   }
   addShape(type: string) {
     this.clearSelection();
@@ -410,6 +440,11 @@ export class WhiteboardPageComponent implements OnInit {
         console.log(component.transformers)
         component.layer.batchDraw();
         component.selectedShape = tr.nodes()[0];
+
+        
+
+        // document.getElementById('audioPlayer').load();
+
       }
       else {
         tr.detach();
@@ -417,6 +452,7 @@ export class WhiteboardPageComponent implements OnInit {
         component.layer.batchDraw();
         component.selectedShape = null;
       }
+
     });
     tr.on('transform', function () {
       var shape = tr.nodes()[0];
@@ -429,6 +465,7 @@ export class WhiteboardPageComponent implements OnInit {
       // shape.scaleY(1);
 
     });
+
   }
   addDeleteListener() {
     const component = this;
@@ -486,9 +523,35 @@ export class WhiteboardPageComponent implements OnInit {
       this.boardId = '' + newUid.id;
       this.boardTitle = mapTitle;
 
-      message = "Plan sauvegardé avec succès";
-      alert(message);
+      var blobToBase64 = blob => {
+        var reader = new FileReader();
+        reader.readAsDataURL(blob);
+        return new Promise(resolve => {
+          reader.onloadend = () => {
+            resolve(reader.result);
+          };
+        });
+      };
+      for (const [key, url] of Object.entries(this.shapeAudio)) {
+        let audio :Audio;
+        let audioEncoded;
+        let blobFromURL = await fetch(url).then(r => r.blob());
+        await blobToBase64(blobFromURL).then(res => {
+          audioEncoded = res;
+        });
+        audio = {
+          idBoard : this.boardId,
+          idShape : key,
+          audioName : this.boardId + '-' + key,
+          audioEncoded : '' + audioEncoded
+        }
+        await this.boardsDbService.saveAudio(audio);
+      }
+
 
+
+        message = "ok";
+        alert(message);
     } catch (error) {
       message = "Une erreur s'est produite lors de la sauvegarde du plan";
       alert(message);