11 #include <mrpt/config.h>    15 #include <mrpt/config.h>    16 #if MRPT_HAS_OPENCV && MRPT_OPENCV_VERSION_NUM<0x240    17 #       undef MRPT_HAS_OPENCV    18 #       define MRPT_HAS_OPENCV 0    22     #include <mrpt/otherlibs/do_opencv_includes.h>    29       if (cameraMatrix.depth() == CV_32F)
    30           init_camera_parameters<float>(cameraMatrix);
    32         init_camera_parameters<double>(cameraMatrix);
    34       number_of_correspondences = std::max(opoints.checkVector(3, CV_32F), opoints.checkVector(3, CV_64F));
    36       pws.resize(3 * number_of_correspondences);
    37       us.resize(2 * number_of_correspondences);
    39       if (opoints.depth() == ipoints.depth())
    41         if (opoints.depth() == CV_32F)
    42           init_points<cv::Point3f,cv::Point2f>(opoints, ipoints);
    44           init_points<cv::Point3d,cv::Point2d>(opoints, ipoints);
    46       else if (opoints.depth() == CV_32F)
    47         init_points<cv::Point3f,cv::Point2d>(opoints, ipoints);
    49         init_points<cv::Point3d,cv::Point2f>(opoints, ipoints);
    51       alphas.resize(4 * number_of_correspondences);
    52       pcs.resize(3 * number_of_correspondences);
    70       cws[0][0] = cws[0][1] = cws[0][2] = 0;
    71       for(
int i = 0; i < number_of_correspondences; i++)
    72         for(
int j = 0; j < 3; j++)
    73           cws[0][j] += pws[3 * i + j];
    75       for(
int j = 0; j < 3; j++)
    76         cws[0][j] /= number_of_correspondences;
    80       CvMat * PW0 = cvCreateMat(number_of_correspondences, 3, CV_64F);
    82       double pw0tpw0[3 * 3], dc[3], uct[3 * 3];
    83       CvMat PW0tPW0 = cvMat(3, 3, CV_64F, pw0tpw0);
    84       CvMat DC      = cvMat(3, 1, CV_64F, dc);
    85       CvMat UCt     = cvMat(3, 3, CV_64F, uct);
    87       for(
int i = 0; i < number_of_correspondences; i++)
    88         for(
int j = 0; j < 3; j++)
    89           PW0->data.db[3 * i + j] = pws[3 * i + j] - cws[0][j];
    91       cvMulTransposed(PW0, &PW0tPW0, 1);
    92       cvSVD(&PW0tPW0, &DC, &UCt, 0, CV_SVD_MODIFY_A | CV_SVD_U_T);
    96       for(
int i = 1; i < 4; i++) {
    97         double k = sqrt(dc[i - 1] / number_of_correspondences);
    98         for(
int j = 0; j < 3; j++)
    99           cws[i][j] = cws[0][j] + k * uct[3 * (i - 1) + j];
   105       double cc[3 * 3], cc_inv[3 * 3];
   106       CvMat CC     = cvMat(3, 3, CV_64F, cc);
   107       CvMat CC_inv = cvMat(3, 3, CV_64F, cc_inv);
   109       for(
int i = 0; i < 3; i++)
   110         for(
int j = 1; j < 4; j++)
   111           cc[3 * i + j - 1] = cws[j][i] - cws[0][i];
   113       cvInvert(&CC, &CC_inv, CV_SVD);
   114       double * ci = cc_inv;
   115       for(
int i = 0; i < number_of_correspondences; i++) {
   116         double * pi = &pws[0] + 3 * i;
   117         double * 
a = &alphas[0] + 4 * i;
   119         for(
int j = 0; j < 3; j++)
   121       ci[3 * j    ] * (pi[0] - cws[0][0]) +
   122       ci[3 * j + 1] * (pi[1] - cws[0][1]) +
   123       ci[3 * j + 2] * (pi[2] - cws[0][2]);
   124         a[0] = 1.0f - 
a[1] - 
a[2] - 
a[3];
   129           const int row, 
const double * as, 
const double u, 
const double v)
   131       double * M1 = M->data.db + 
row * 12;
   132       double * M2 = M1 + 12;
   134       for(
int i = 0; i < 4; i++) {
   135         M1[3 * i    ] = as[i] * fu;
   137         M1[3 * i + 2] = as[i] * (uc - u);
   140         M2[3 * i + 1] = as[i] * fv;
   141         M2[3 * i + 2] = as[i] * (vc - 
v);
   147       for(
int i = 0; i < 4; i++)
   148         ccs[i][0] = ccs[i][1] = ccs[i][2] = 0.0f;
   150       for(
int i = 0; i < 4; i++) {
   151         const double * 
v = ut + 12 * (11 - i);
   152         for(
int j = 0; j < 4; j++)
   153           for(
int k = 0; k < 3; k++)
   154       ccs[j][k] += betas[i] * 
v[3 * j + k];
   160       for(
int i = 0; i < number_of_correspondences; i++) {
   161         double * 
a = &alphas[0] + 4 * i;
   162         double * pc = &pcs[0] + 3 * i;
   164         for(
int j = 0; j < 3; j++)
   165           pc[j] = 
a[0] * ccs[0][j] + 
a[1] * ccs[1][j] + 
a[2] * ccs[2][j] + 
a[3] * ccs[3][j];
   171       choose_control_points();
   172       compute_barycentric_coordinates();
   174       CvMat * M = cvCreateMat(2 * number_of_correspondences, 12, CV_64F);
   176       for(
int i = 0; i < number_of_correspondences; i++)
   177         fill_M(M, 2 * i, &alphas[0] + 4 * i, us[2 * i], us[2 * i + 1]);
   179       double mtm[12 * 12], d[12], ut[12 * 12];
   180       CvMat MtM = cvMat(12, 12, CV_64F, mtm);
   181       CvMat D   = cvMat(12,  1, CV_64F, d);
   182       CvMat Ut  = cvMat(12, 12, CV_64F, ut);
   184       cvMulTransposed(M, &MtM, 1);
   185       cvSVD(&MtM, &D, &Ut, 0, CV_SVD_MODIFY_A | CV_SVD_U_T);
   188       double l_6x10[6 * 10], rho[6];
   189       CvMat L_6x10 = cvMat(6, 10, CV_64F, l_6x10);
   190       CvMat Rho    = cvMat(6,  1, CV_64F, rho);
   192       compute_L_6x10(ut, l_6x10);
   195       double Betas[4][4], rep_errors[4];
   196       double Rs[4][3][3], ts[4][3];
   198       find_betas_approx_1(&L_6x10, &Rho, Betas[1]);
   199       gauss_newton(&L_6x10, &Rho, Betas[1]);
   200       rep_errors[1] = compute_R_and_t(ut, Betas[1], Rs[1], ts[1]);
   202       find_betas_approx_2(&L_6x10, &Rho, Betas[2]);
   203       gauss_newton(&L_6x10, &Rho, Betas[2]);
   204       rep_errors[2] = compute_R_and_t(ut, Betas[2], Rs[2], ts[2]);
   206       find_betas_approx_3(&L_6x10, &Rho, Betas[3]);
   207       gauss_newton(&L_6x10, &Rho, Betas[3]);
   208       rep_errors[3] = compute_R_and_t(ut, Betas[3], Rs[3], ts[3]);
   211       if (rep_errors[2] < rep_errors[1]) N = 2;
   212       if (rep_errors[3] < rep_errors[N]) N = 3;
   214       cv::Mat(3, 1, CV_64F, ts[N]).copyTo(
t);
   215       cv::Mat(3, 3, CV_64F, Rs[N]).copyTo(
R);
   219           double R_dst[3][3], 
double t_dst[3])
   221       for(
int i = 0; i < 3; i++) {
   222         for(
int j = 0; j < 3; j++)
   223           R_dst[i][j] = R_src[i][j];
   231         (p1[0] - p2[0]) * (p1[0] - p2[0]) +
   232         (p1[1] - p2[1]) * (p1[1] - p2[1]) +
   233         (p1[2] - p2[2]) * (p1[2] - p2[2]);
   243       double pc0[3], pw0[3];
   245       pc0[0] = pc0[1] = pc0[2] = 0.0;
   246       pw0[0] = pw0[1] = pw0[2] = 0.0;
   248       for(
int i = 0; i < number_of_correspondences; i++) {
   249         const double * pc = &pcs[3 * i];
   250         const double * pw = &pws[3 * i];
   252         for(
int j = 0; j < 3; j++) {
   257       for(
int j = 0; j < 3; j++) {
   258         pc0[j] /= number_of_correspondences;
   259         pw0[j] /= number_of_correspondences;
   262       double abt[3 * 3], abt_d[3], abt_u[3 * 3], abt_v[3 * 3];
   263       CvMat ABt   = cvMat(3, 3, CV_64F, abt);
   264       CvMat ABt_D = cvMat(3, 1, CV_64F, abt_d);
   265       CvMat ABt_U = cvMat(3, 3, CV_64F, abt_u);
   266       CvMat ABt_V = cvMat(3, 3, CV_64F, abt_v);
   269       for(
int i = 0; i < number_of_correspondences; i++) {
   270         double * pc = &pcs[3 * i];
   271         double * pw = &pws[3 * i];
   273         for(
int j = 0; j < 3; j++) {
   274           abt[3 * j    ] += (pc[j] - pc0[j]) * (pw[0] - pw0[0]);
   275           abt[3 * j + 1] += (pc[j] - pc0[j]) * (pw[1] - pw0[1]);
   276           abt[3 * j + 2] += (pc[j] - pc0[j]) * (pw[2] - pw0[2]);
   280       cvSVD(&ABt, &ABt_D, &ABt_U, &ABt_V, CV_SVD_MODIFY_A);
   282       for(
int i = 0; i < 3; i++)
   283         for(
int j = 0; j < 3; j++)
   284           R[i][j] = dot(abt_u + 3 * i, abt_v + 3 * j);
   287         R[0][0] * 
R[1][1] * 
R[2][2] + 
R[0][1] * 
R[1][2] * 
R[2][0] + 
R[0][2] * 
R[1][0] * 
R[2][1] -
   288         R[0][2] * 
R[1][1] * 
R[2][0] - 
R[0][1] * 
R[1][0] * 
R[2][2] - 
R[0][0] * 
R[1][2] * 
R[2][1];
   296       t[0] = pc0[0] - dot(
R[0], pw0);
   297       t[1] = pc0[1] - dot(
R[1], pw0);
   298       t[2] = pc0[2] - dot(
R[2], pw0);
   304         for(
int i = 0; i < 4; i++)
   305           for(
int j = 0; j < 3; j++)
   306       ccs[i][j] = -ccs[i][j];
   308         for(
int i = 0; i < number_of_correspondences; i++) {
   309           pcs[3 * i    ] = -pcs[3 * i];
   310           pcs[3 * i + 1] = -pcs[3 * i + 1];
   311           pcs[3 * i + 2] = -pcs[3 * i + 2];
   317                double R[3][3], 
double t[3])
   319       compute_ccs(betas, ut);
   324       estimate_R_and_t(
R, 
t);
   326       return reprojection_error(
R, 
t);
   333       for(
int i = 0; i < number_of_correspondences; i++) {
   334         double * pw = &pws[3 * i];
   335         double Xc = dot(
R[0], pw) + 
t[0];
   336         double Yc = dot(
R[1], pw) + 
t[1];
   337         double inv_Zc = 1.0 / (dot(
R[2], pw) + 
t[2]);
   338         double ue = uc + fu * Xc * inv_Zc;
   339         double ve = vc + fv * Yc * inv_Zc;
   340         double u = us[2 * i], 
v = us[2 * i + 1];
   342         sum2 += sqrt( (u - ue) * (u - ue) + (
v - ve) * (
v - ve) );
   345       return sum2 / number_of_correspondences;
   354       double l_6x4[6 * 4], b4[4];
   355       CvMat L_6x4 = cvMat(6, 4, CV_64F, l_6x4);
   356       CvMat B4    = cvMat(4, 1, CV_64F, b4);
   358       for(
int i = 0; i < 6; i++) {
   359         cvmSet(&L_6x4, i, 0, cvmGet(L_6x10, i, 0));
   360         cvmSet(&L_6x4, i, 1, cvmGet(L_6x10, i, 1));
   361         cvmSet(&L_6x4, i, 2, cvmGet(L_6x10, i, 3));
   362         cvmSet(&L_6x4, i, 3, cvmGet(L_6x10, i, 6));
   365       cvSolve(&L_6x4, Rho, &B4, CV_SVD);
   368         betas[0] = sqrt(-b4[0]);
   369         betas[1] = -b4[1] / betas[0];
   370         betas[2] = -b4[2] / betas[0];
   371         betas[3] = -b4[3] / betas[0];
   373         betas[0] = sqrt(b4[0]);
   374         betas[1] = b4[1] / betas[0];
   375         betas[2] = b4[2] / betas[0];
   376         betas[3] = b4[3] / betas[0];
   386       double l_6x3[6 * 3], 
b3[3];
   387       CvMat L_6x3  = cvMat(6, 3, CV_64F, l_6x3);
   388       CvMat B3     = cvMat(3, 1, CV_64F, 
b3);
   390       for(
int i = 0; i < 6; i++) {
   391         cvmSet(&L_6x3, i, 0, cvmGet(L_6x10, i, 0));
   392         cvmSet(&L_6x3, i, 1, cvmGet(L_6x10, i, 1));
   393         cvmSet(&L_6x3, i, 2, cvmGet(L_6x10, i, 2));
   396       cvSolve(&L_6x3, Rho, &B3, CV_SVD);
   399         betas[0] = sqrt(-
b3[0]);
   400         betas[1] = (
b3[2] < 0) ? sqrt(-
b3[2]) : 0.0;
   402         betas[0] = sqrt(
b3[0]);
   403         betas[1] = (
b3[2] > 0) ? sqrt(
b3[2]) : 0.0;
   406       if (
b3[1] < 0) betas[0] = -betas[0];
   418       double l_6x5[6 * 5], b5[5];
   419       CvMat L_6x5 = cvMat(6, 5, CV_64F, l_6x5);
   420       CvMat B5    = cvMat(5, 1, CV_64F, b5);
   422       for(
int i = 0; i < 6; i++) {
   423         cvmSet(&L_6x5, i, 0, cvmGet(L_6x10, i, 0));
   424         cvmSet(&L_6x5, i, 1, cvmGet(L_6x10, i, 1));
   425         cvmSet(&L_6x5, i, 2, cvmGet(L_6x10, i, 2));
   426         cvmSet(&L_6x5, i, 3, cvmGet(L_6x10, i, 3));
   427         cvmSet(&L_6x5, i, 4, cvmGet(L_6x10, i, 4));
   430       cvSolve(&L_6x5, Rho, &B5, CV_SVD);
   433         betas[0] = sqrt(-b5[0]);
   434         betas[1] = (b5[2] < 0) ? sqrt(-b5[2]) : 0.0;
   436         betas[0] = sqrt(b5[0]);
   437         betas[1] = (b5[2] > 0) ? sqrt(b5[2]) : 0.0;
   439       if (b5[1] < 0) betas[0] = -betas[0];
   440       betas[2] = b5[3] / betas[0];
   455       for(
int i = 0; i < 4; i++) {
   457         for(
int j = 0; j < 6; j++) {
   458           dv[i][j][0] = 
v[i][3 * 
a    ] - 
v[i][3 * 
b];
   459           dv[i][j][1] = 
v[i][3 * 
a + 1] - 
v[i][3 * 
b + 1];
   460           dv[i][j][2] = 
v[i][3 * 
a + 2] - 
v[i][3 * 
b + 2];
   470       for(
int i = 0; i < 6; i++) {
   471         double * 
row = l_6x10 + 10 * i;
   473         row[0] =        dot(dv[0][i], dv[0][i]);
   474         row[1] = 2.0f * dot(dv[0][i], dv[1][i]);
   475         row[2] =        dot(dv[1][i], dv[1][i]);
   476         row[3] = 2.0f * dot(dv[0][i], dv[2][i]);
   477         row[4] = 2.0f * dot(dv[1][i], dv[2][i]);
   478         row[5] =        dot(dv[2][i], dv[2][i]);
   479         row[6] = 2.0f * dot(dv[0][i], dv[3][i]);
   480         row[7] = 2.0f * dot(dv[1][i], dv[3][i]);
   481         row[8] = 2.0f * dot(dv[2][i], dv[3][i]);
   482         row[9] =        dot(dv[3][i], dv[3][i]);
   488       rho[0] = dist2(cws[0], cws[1]);
   489       rho[1] = dist2(cws[0], cws[2]);
   490       rho[2] = dist2(cws[0], cws[3]);
   491       rho[3] = dist2(cws[1], cws[2]);
   492       rho[4] = dist2(cws[1], cws[3]);
   493       rho[5] = dist2(cws[2], cws[3]);
   497               const double betas[4], CvMat * A, CvMat * 
b)
   499       for(
int i = 0; i < 6; i++) {
   500         const double * rowL = l_6x10 + i * 10;
   501         double * rowA = A->data.db + i * 4;
   503         rowA[0] = 2 * rowL[0] * betas[0] +     rowL[1] * betas[1] +     rowL[3] * betas[2] +     rowL[6] * betas[3];
   504         rowA[1] =     rowL[1] * betas[0] + 2 * rowL[2] * betas[1] +     rowL[4] * betas[2] +     rowL[7] * betas[3];
   505         rowA[2] =     rowL[3] * betas[0] +     rowL[4] * betas[1] + 2 * rowL[5] * betas[2] +     rowL[8] * betas[3];
   506         rowA[3] =     rowL[6] * betas[0] +     rowL[7] * betas[1] +     rowL[8] * betas[2] + 2 * rowL[9] * betas[3];
   508         cvmSet(
b, i, 0, rho[i] -
   510           rowL[0] * betas[0] * betas[0] +
   511           rowL[1] * betas[0] * betas[1] +
   512           rowL[2] * betas[1] * betas[1] +
   513           rowL[3] * betas[0] * betas[2] +
   514           rowL[4] * betas[1] * betas[2] +
   515           rowL[5] * betas[2] * betas[2] +
   516           rowL[6] * betas[0] * betas[3] +
   517           rowL[7] * betas[1] * betas[3] +
   518           rowL[8] * betas[2] * betas[3] +
   519           rowL[9] * betas[3] * betas[3]
   526       const int iterations_number = 5;
   528       double a[6*4], 
b[6], 
x[4];
   529       CvMat A = cvMat(6, 4, CV_64F, 
a);
   530       CvMat B = cvMat(6, 1, CV_64F, 
b);
   531       CvMat X = cvMat(4, 1, CV_64F, 
x);
   533       for(
int k = 0; k < iterations_number; k++)
   535         compute_A_and_b_gauss_newton(L_6x10->data.db, Rho->data.db,
   537         qr_solve(&A, &B, &X);
   538         for(
int i = 0; i < 4; i++)
   545       const int nr = A->rows;
   546       const int nc = A->cols;
   548       if (max_nr != 0 && max_nr < nr)
   560       double * pA = A->data.db, * ppAkk = pA;
   561       for(
int k = 0; k < nc; k++)
   563         double * ppAik1 = ppAkk, eta = fabs(*ppAik1);
   564         for(
int i = k + 1; i < nr; i++)
   566           double elt = fabs(*ppAik1);
   567           if (eta < elt) eta = elt;
   578           double * ppAik2 = ppAkk, sum2 = 0.0, inv_eta = 1. / eta;
   579           for(
int i = k; i < nr; i++)
   582             sum2 += *ppAik2 * *ppAik2;
   585           double sigma = sqrt(sum2);
   589           A1[k] = sigma * *ppAkk;
   590           A2[k] = -eta * sigma;
   591           for(
int j = k + 1; j < nc; j++)
   593             double * ppAik = ppAkk, 
sum = 0;
   594             for(
int i = k; i < nr; i++)
   596               sum += *ppAik * ppAik[j - k];
   599             double tau = 
sum / 
A1[k];
   601             for(
int i = k; i < nr; i++)
   603               ppAik[j - k] -= tau * *ppAik;
   612       double * ppAjj = pA, * pb = 
b->data.db;
   613       for(
int j = 0; j < nc; j++)
   615         double * ppAij = ppAjj, tau = 0;
   616         for(
int i = j; i < nr; i++)
   618           tau += *ppAij * pb[i];
   623         for(
int i = j; i < nr; i++)
   625           pb[i] -= tau * *ppAij;
   632       double * pX = X->data.db;
   633       pX[nc - 1] = pb[nc - 1] / A2[nc - 1];
   634       for(
int i = nc - 2; i >= 0; i--)
   636         double * ppAij = pA + i * nc + (i + 1), 
sum = 0;
   638         for(
int j = i + 1; j < nc; j++)
   640           sum += *ppAij * pX[j];
   643         pX[i] = (pb[i] - 
sum) / A2[i];
 EIGEN_STRONG_INLINE Scalar det() const
 
void compute_A_and_b_gauss_newton(const double *l_6x10, const double *rho, const double cb[4], CvMat *A, CvMat *b)
Internal function. 
 
void compute_barycentric_coordinates(void)
Convert from object space to relative object space (Barycentric coordinates) 
 
void choose_control_points(void)
Function to select 4 control points from n points. 
 
void find_betas_approx_3(const CvMat *L_6x10, const CvMat *Rho, double *betas)
Internal function. 
 
void compute_pose(cv::Mat &R, cv::Mat &t)
OpenCV wrapper to compute pose. 
 
void compute_rho(double *rho)
Get distances between all object points taken 2 at a time(nC2) 
 
~epnp()
Destructor for EPnP class. 
 
double reprojection_error(const double R[3][3], const double t[3])
Function to compute reprojection error. 
 
double dist2(const double *p1, const double *p2)
Squared distance between two vectors. 
 
void gauss_newton(const CvMat *L_6x10, const CvMat *Rho, double current_betas[4])
Gauss Newton iterative algorithm. 
 
void copy_R_and_t(const double R_dst[3][3], const double t_dst[3], double R_src[3][3], double t_src[3])
Copy function of output result. 
 
void solve_for_sign(void)
Internal function. 
 
void fill_M(CvMat *M, const int row, const double *alphas, const double u, const double v)
Generate the Matrix M. 
 
double dot(const double *v1, const double *v2)
Dot product of two OpenCV vectors. 
 
CONTAINER::Scalar sum(const CONTAINER &v)
Computes the sum of all the elements. 
 
Efficient PnP - Eigen Wrapper for OpenCV calib3d implementation. 
 
void qr_solve(CvMat *A, CvMat *b, CvMat *X)
QR optimization algorithm. 
 
double A1
UTC constant and 1st order terms. 
 
void estimate_R_and_t(double R[3][3], double t[3])
Helper function to  compute_R_and_t() 
 
void find_betas_approx_2(const CvMat *L_6x10, const CvMat *Rho, double *betas)
Internal function. 
 
GLenum GLenum GLvoid * row
 
double compute_R_and_t(const double *ut, const double *betas, double R[3][3], double t[3])
Function to compute pose. 
 
void compute_ccs(const double *betas, const double *ut)
Internal function. 
 
void compute_pcs(void)
Internal function. 
 
GLfloat GLfloat GLfloat v2
 
epnp(const cv::Mat &cameraMatrix, const cv::Mat &opoints, const cv::Mat &ipoints)
Constructor for EPnP class. 
 
GLubyte GLubyte GLubyte a
 
void compute_L_6x10(const double *ut, double *l_6x10)
Internal function. 
 
void find_betas_approx_1(const CvMat *L_6x10, const CvMat *Rho, double *betas)
Internal function.