From f4e8f3f1aaf6b4008bb208e9ecf9cdc2bf4f2a27 Mon Sep 17 00:00:00 2001
From: "simon.fanetti" <simon.fanetti@etu.hesge.ch>
Date: Mon, 29 Jun 2020 22:12:43 +0200
Subject: [PATCH] fixe cropping mask + add button Clear and Reset in setup app
---
app/SandboxSetup/croppingmask.cpp | 41 +++++++++++++++++++++++++++----
app/SandboxSetup/croppingmask.h | 10 ++++++--
app/SandboxSetup/croppingmask.ui | 26 ++++++++++++++++++++
inc/beamer.h | 6 ++---
inc/projection.h | 1 +
inc/sandboxSetup.h | 2 ++
src/components/beamer.cpp | 6 ++---
src/components/camera.cpp | 4 +--
src/components/projection.cpp | 13 +++++++++-
src/lib/sandboxSetup.cpp | 35 ++++++++++++++++++++++----
10 files changed, 122 insertions(+), 22 deletions(-)
diff --git a/app/SandboxSetup/croppingmask.cpp b/app/SandboxSetup/croppingmask.cpp
index d68e181..cf84ff2 100644
--- a/app/SandboxSetup/croppingmask.cpp
+++ b/app/SandboxSetup/croppingmask.cpp
@@ -15,7 +15,6 @@ CroppingMask::CroppingMask(SandboxSetup *sandbox, MonitorGui *_mg, QWidget *pare
blueScreen = new QtFullScreen(mg->getResolution(), true, this);
- // TODO : try to load cropping mask from file
timer = new QTimer(this);
connect(timer, &QTimer::timeout, this, &CroppingMask::refreshFrame);
}
@@ -36,6 +35,7 @@ void CroppingMask::valideRoutine(){
std::vector<cv::Point> rectPoints = getRectPoints();
cv::Mat depthFrame = setup->getCamera()->getDepthFrame();
+ // TODO : center should be defined by the quadrilateral from rectPoints
setup->setupAdjustMatrix(rectPoints, center);
setup->setupCroppingMask(rectPoints);
setup->getProjection()->setDistanceTopSandbox(depthFrame.at<float>(center));
@@ -72,13 +72,26 @@ void CroppingMask::init(){
setup->getCamera()->capture();
cameraColoredFrame = setup->getCamera()->getColorFrame();
- // no config found
+ // load from config
if(rectPoints.empty()){
- if(!maskValideInFrame(&cameraColoredFrame)){
+ if(!setup->loadCroppingMask()){
+ cv::Rect mask = setup->getCamera()->getCroppingMask();
+ std::vector<cv::Point2i> pts = { cv::Point2i(mask.x, mask.y),
+ cv::Point2i(mask.x, mask.y+mask.height),
+ cv::Point2i(mask.x+mask.width, mask.y+mask.height),
+ cv::Point2i(mask.x+mask.width, mask.y) };
+ for(cv::Point2i pt : pts){
+ loadedMask.push_back(setup->getProjection()->revertRotatePixel(pt));
+ }
+ rectPoints = loadedMask;
+ }
+ }
+
+ // no config found
+ if(rectPoints.empty() || !maskValideInFrame(&cameraColoredFrame)){
float y = cameraColoredFrame.size().height;
float x = cameraColoredFrame.size().width;
- rectPoints = std::vector<cv::Point>{ cv::Point(1.0/4*x, 1.0/4*y), cv::Point(1.0/4*x, 3.0/4*y), cv::Point(3.0/4*x, 3.0/4*y), cv::Point(3.0/4*x, 1.0/4*y) };
- }
+ rectPoints = std::vector<cv::Point2i>{ cv::Point2i(1.0/4*x, 1.0/4*y), cv::Point2i(1.0/4*x, 3.0/4*y), cv::Point2i(3.0/4*x, 3.0/4*y), cv::Point2i(3.0/4*x, 1.0/4*y) };
}
maskEdit->updateFrame(&cameraColoredFrame, &rectPoints);
@@ -106,3 +119,21 @@ bool CroppingMask::maskValideInFrame(cv::Mat *rgb){
return true;
}
+
+void CroppingMask::on_btnClear_clicked()
+{
+ if(cameraColoredFrame.size().height != 0 && cameraColoredFrame.size().width != 0){
+ float y = cameraColoredFrame.size().height;
+ float x = cameraColoredFrame.size().width;
+ rectPoints = std::vector<cv::Point2i>{ cv::Point2i(1.0/4*x, 1.0/4*y), cv::Point2i(1.0/4*x, 3.0/4*y), cv::Point2i(3.0/4*x, 3.0/4*y), cv::Point2i(3.0/4*x, 1.0/4*y) };
+ maskEdit->updateFrame(&rectPoints);
+ }
+}
+
+void CroppingMask::on_btnReset_clicked()
+{
+ if(!loadedMask.empty()){
+ rectPoints = loadedMask;
+ maskEdit->updateFrame(&rectPoints);
+ }
+}
diff --git a/app/SandboxSetup/croppingmask.h b/app/SandboxSetup/croppingmask.h
index 33853a6..6fb694c 100644
--- a/app/SandboxSetup/croppingmask.h
+++ b/app/SandboxSetup/croppingmask.h
@@ -21,7 +21,7 @@ class CroppingMask : public SubApp
public:
explicit CroppingMask(SandboxSetup *sandbox, MonitorGui *_mg, QWidget *parent = 0);
~CroppingMask();
- std::vector<cv::Point> getRectPoints(){ return rectPoints; };
+ std::vector<cv::Point2i> getRectPoints(){ return rectPoints; };
void valideRoutine();
void cancelRoutine();
@@ -29,10 +29,16 @@ protected:
void showEvent(QShowEvent *event);
void closeEvent(QCloseEvent *event);
+private slots:
+ void on_btnClear_clicked();
+
+ void on_btnReset_clicked();
+
private:
Ui::CroppingMask *ui;
SandboxSetup *setup;
- std::vector<cv::Point> rectPoints;
+ std::vector<cv::Point2i> rectPoints;
+ std::vector<cv::Point2i> loadedMask;
cv::Mat cameraColoredFrame;
MaskEdit *maskEdit;
MonitorGui *mg;
diff --git a/app/SandboxSetup/croppingmask.ui b/app/SandboxSetup/croppingmask.ui
index 2f4ec8e..f7d7133 100644
--- a/app/SandboxSetup/croppingmask.ui
+++ b/app/SandboxSetup/croppingmask.ui
@@ -77,6 +77,32 @@ p, li { white-space: pre-wrap; }
<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">At this point, the beamer and the blue screen should be set correctly. So now we want to determine the edges of the blue screen. To do that, be sure to have a flat surface to have a clean blue rectangle projected. (ie. you can use a cardboard as support). Then drag the corners of the green rectangle to match the bluescreen's corners.</p></body></html></string>
</property>
</widget>
+ <widget class="QPushButton" name="btnClear">
+ <property name="geometry">
+ <rect>
+ <x>520</x>
+ <y>40</y>
+ <width>89</width>
+ <height>25</height>
+ </rect>
+ </property>
+ <property name="text">
+ <string>Clear</string>
+ </property>
+ </widget>
+ <widget class="QPushButton" name="btnReset">
+ <property name="geometry">
+ <rect>
+ <x>520</x>
+ <y>80</y>
+ <width>89</width>
+ <height>25</height>
+ </rect>
+ </property>
+ <property name="text">
+ <string>Reset</string>
+ </property>
+ </widget>
</widget>
<resources/>
<connections/>
diff --git a/inc/beamer.h b/inc/beamer.h
index 3a20048..1832820 100644
--- a/inc/beamer.h
+++ b/inc/beamer.h
@@ -9,11 +9,9 @@
class Beamer{
private:
cv::Point3f beamerPosition;
- cv::Size resolution = cv::Size(256,144);
+ cv::Size resolution;
FrameProcessProfil *profil;
-
- //cv::Point3f intersection(cv::Vec3f v1, cv::Point3f p1, cv::Vec3f v2, cv::Point3f p2, cv::Vec3f v3, cv::Point3f p3, bool &isFound);
-
+
public:
Beamer();
~Beamer();
diff --git a/inc/projection.h b/inc/projection.h
index ff33e10..6d906b5 100644
--- a/inc/projection.h
+++ b/inc/projection.h
@@ -26,6 +26,7 @@ class Projection{
float getDistanceTopSandbox(){ return distanceTopSandbox; };
cv::Point2i rotatePixel(cv::Point2i pixel);
+ cv::Point2i revertRotatePixel(cv::Point2i pixel);
void adjustFrame(cv::Mat depth, cv::Mat src, cv::Mat &dst, Camera *camera, cv::Point3f beamer_pos);
void printAdjustingMatrix();
diff --git a/inc/sandboxSetup.h b/inc/sandboxSetup.h
index 3d2e370..94cbf17 100644
--- a/inc/sandboxSetup.h
+++ b/inc/sandboxSetup.h
@@ -31,6 +31,8 @@ class SandboxSetup{
int loadFrameProcessProfilFrom(char *path);
int loadFrameProcessProfil();
+ int loadCroppingMask(char *path);
+ int loadCroppingMask();
// edit variables of config => not persistant
void setupAdjustMatrix(std::vector<cv::Point> rectPoints, cv::Point center);
diff --git a/src/components/beamer.cpp b/src/components/beamer.cpp
index b5bbf12..7f92c3b 100644
--- a/src/components/beamer.cpp
+++ b/src/components/beamer.cpp
@@ -4,6 +4,7 @@
Beamer::Beamer(){
beamerPosition = cv::Point3f(0.0f, 0.265f, -0.205f);
+ resolution = cv::Size(256,144);
profil = new FrameProcessProfil();
}
@@ -218,11 +219,10 @@ cv::Point3d Beamer::approximatePosition(std::vector<cv::Point3d> *bases, std::ve
}
cv::Point3d beamerPoint(0.0, 0.0, 0.0);
- for (unsigned int i = 0; i < beamerPoints.size(); i++)
- {
+ for (unsigned int i = 0; i < beamerPoints.size(); i++){
beamerPoint += beamerPoints[i];
}
- beamerPoint /= 6.0;
+ beamerPoint /= (double)beamerPoints.size();
return beamerPoint;
}
\ No newline at end of file
diff --git a/src/components/camera.cpp b/src/components/camera.cpp
index d75ae16..dda660f 100644
--- a/src/components/camera.cpp
+++ b/src/components/camera.cpp
@@ -91,7 +91,7 @@ void Camera::capture(){
}
}
-// get the coordinates of the pixel matching the point relative to the real world
+// Get the coordinates of the pixel matching the point relative to the real world
cv::Point2i Camera::projectPointToPixel(cv::Point3f point3D){
float point[3] = {point3D.x, point3D.y, point3D.z};
@@ -101,7 +101,7 @@ cv::Point2i Camera::projectPointToPixel(cv::Point3f point3D){
return cv::Point2i(pixel[0], pixel[1]);
}
-// get the point relative to the real world matching the coordinates of the pixel
+// Get the point relative to the real world matching the coordinates of the pixel
cv::Point3f Camera::deprojectPixelToPoint(float coord[], float z){
float p[3];
diff --git a/src/components/projection.cpp b/src/components/projection.cpp
index bc06993..77b8a51 100644
--- a/src/components/projection.cpp
+++ b/src/components/projection.cpp
@@ -18,6 +18,16 @@ cv::Point2i Projection::rotatePixel(cv::Point2i pixel){
return cv::Point2i(tmp.at<cv::Vec2f>(0, 0));
}
+cv::Point2i Projection::revertRotatePixel(cv::Point2i pixel){
+
+ cv::Mat tmp = (cv::Mat_<cv::Vec2f>(1, 1) << cv::Vec2f(pixel.x, pixel.y));
+ cv::Mat invMat;
+ cv::invertAffineTransform(adjustingMatrix, invMat);
+ cv::transform(tmp, tmp, invMat);
+ return cv::Point2i(tmp.at<cv::Vec2f>(0, 0));
+}
+
+
// Adjust the projected frame with the topology from the camera to the beamer POV
void Projection::adjustFrame(cv::Mat depth, cv::Mat src, cv::Mat &dst, Camera *camera, cv::Point3f beamer_pos){
@@ -36,8 +46,9 @@ void Projection::adjustFrame(cv::Mat depth, cv::Mat src, cv::Mat &dst, Camera *c
filterHighestDeprojectedPoints(depth, pixelsDeprojectMap, pixelsDeprojectHighestMap);
+ // TODO : Holefilling method
+
buildFrame(depth, pixelsDeprojectHighestMap, src, dst);
-
cv::resize(dst, dst, dst_size);
cv::warpAffine(dst, dst, adjustingMatrix, dst.size());
diff --git a/src/lib/sandboxSetup.cpp b/src/lib/sandboxSetup.cpp
index caf24db..6b969a3 100644
--- a/src/lib/sandboxSetup.cpp
+++ b/src/lib/sandboxSetup.cpp
@@ -46,6 +46,14 @@ int SandboxSetup::saveConfig(){
return saveConfigInto(defaultConfigFilePath);
}
+int SandboxSetup::loadCroppingMask(char *path){
+ return SandboxConfig::loadCroppingMaskFrom(path, camera);
+}
+
+int SandboxSetup::loadCroppingMask(){
+ return loadCroppingMask(defaultConfigFilePath);
+}
+
int SandboxSetup::loadFrameProcessProfilFrom(char *path){
return SandboxConfig::loadFrameProcessProfil(path, beamer->getProfil());
}
@@ -54,7 +62,14 @@ int SandboxSetup::loadFrameProcessProfil(){
return loadFrameProcessProfilFrom(defaultConfigFilePath);
}
-
+/* Assuming points positions are :
+ * pts[0] : top left
+ * pts[1] : bottom left
+ * pts[2] : bottom right
+ * pts[3] : top right
+ *
+ * center : center of the rotation in the projected frame
+ */
void SandboxSetup::setupAdjustMatrix(std::vector<cv::Point> rectPoints, cv::Point center){
// Set adjusting matrix for the projection
@@ -64,13 +79,23 @@ void SandboxSetup::setupAdjustMatrix(std::vector<cv::Point> rectPoints, cv::Poin
projection->setAdjustingMatrix(matRotation);
}
-void SandboxSetup::setupCroppingMask(std::vector<cv::Point> rectPoints){
+/* Assuming points positions are :
+ * pts[0] : top left
+ * pts[1] : bottom left
+ * pts[2] : bottom right
+ * pts[3] : top right
+ */
+void SandboxSetup::setupCroppingMask(std::vector<cv::Point2i> rectPoints){
// Set cropping mask
int widthTop = rectPoints[3].x - rectPoints[0].x;
- cv::Size rectSize = cv::Size(widthTop, cvRound(widthTop / 1.33333) + 5);
- cv::Point p = projection->rotatePixel(rectPoints[0]);
- camera->setCroppingMask(cv::Rect(p, rectSize)); // croppingMask
+ cv::Point2i pbl = projection->rotatePixel(rectPoints[1]);
+ cv::Point2i pbr = projection->rotatePixel(rectPoints[2]);
+ cv::Point2i pbot = (pbl.y < pbr.y) ? pbl : pbr;
+ cv::Point2i ptop = projection->rotatePixel(rectPoints[0]);
+
+ cv::Size rectSize = cv::Size(widthTop, pbot.y - rectPoints[0].y);
+ camera->setCroppingMask(cv::Rect(ptop, rectSize)); // croppingMask
}
--
GitLab