libvisiontransfer  8.1.0
reconstruct3d.cpp
1 /*******************************************************************************
2  * Copyright (c) 2020 Nerian Vision GmbH
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a copy
5  * of this software and associated documentation files (the "Software"), to deal
6  * in the Software without restriction, including without limitation the rights
7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8  * copies of the Software, and to permit persons to whom the Software is
9  * furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *******************************************************************************/
14 
15 #include "reconstruct3d.h"
16 #include "visiontransfer/alignedallocator.h"
17 #include <vector>
18 #include <cstring>
19 #include <algorithm>
20 #include <fstream>
21 #include <stdexcept>
22 #include <cmath>
23 
24 // SIMD Headers
25 #ifdef __AVX2__
26 #include <immintrin.h>
27 #elif __SSE2__
28 #include <emmintrin.h>
29 #endif
30 
31 using namespace std;
32 using namespace visiontransfer;
33 using namespace visiontransfer::internal;
34 
35 namespace visiontransfer {
36 
37 /*************** Pimpl class containing all private members ***********/
38 
39 class Reconstruct3D::Pimpl {
40 public:
41  Pimpl();
42 
43  float* createPointMap(const unsigned short* dispMap, int width, int height,
44  int rowStride, const float* q, unsigned short minDisparity);
45 
46  float* createPointMap(const ImageSet& imageSet, unsigned short minDisparity);
47 
48  void projectSinglePoint(int imageX, int imageY, unsigned short disparity, const float* q,
49  float& pointX, float& pointY, float& pointZ);
50 
51  void writePlyFile(const char* file, const unsigned short* dispMap,
52  const unsigned char* image, int width, int height, ImageSet::ImageFormat format,
53  int dispRowStride, int imageRowStride, const float* q,
54  double maxZ, bool binary);
55 
56  void writePlyFile(const char* file, const ImageSet& imageSet,
57  double maxZ, bool binary);
58 
59 private:
60  std::vector<float, AlignedAllocator<float> > pointMap;
61 
62  float* createPointMapFallback(const unsigned short* dispMap, int width, int height,
63  int rowStride, const float* q, unsigned short minDisparity);
64 
65  float* createPointMapSSE2(const unsigned short* dispMap, int width, int height,
66  int rowStride, const float* q, unsigned short minDisparity);
67 
68  float* createPointMapAVX2(const unsigned short* dispMap, int width, int height,
69  int rowStride, const float* q, unsigned short minDisparity);
70 };
71 
72 /******************** Stubs for all public members ********************/
73 
74 Reconstruct3D::Reconstruct3D()
75  :pimpl(new Pimpl) {
76 }
77 
78 Reconstruct3D::~Reconstruct3D() {
79  delete pimpl;
80 }
81 
82 float* Reconstruct3D::createPointMap(const unsigned short* dispMap, int width, int height,
83  int rowStride, const float* q, unsigned short minDisparity) {
84  return pimpl->createPointMap(dispMap, width, height, rowStride, q, minDisparity);
85 }
86 
87 float* Reconstruct3D::createPointMap(const ImageSet& imageSet, unsigned short minDisparity) {
88  return pimpl->createPointMap(imageSet, minDisparity);
89 }
90 
91 void Reconstruct3D::projectSinglePoint(int imageX, int imageY, unsigned short disparity,
92  const float* q, float& pointX, float& pointY, float& pointZ) {
93  pimpl->projectSinglePoint(imageX, imageY, disparity, q, pointX, pointY, pointZ);
94 }
95 
96 void Reconstruct3D::writePlyFile(const char* file, const unsigned short* dispMap,
97  const unsigned char* image, int width, int height, ImageSet::ImageFormat format, int dispRowStride,
98  int imageRowStride, const float* q, double maxZ, bool binary) {
99  pimpl->writePlyFile(file, dispMap, image, width, height, format, dispRowStride,
100  imageRowStride, q, maxZ, binary);
101 }
102 
103 void Reconstruct3D::writePlyFile(const char* file, const ImageSet& imageSet,
104  double maxZ, bool binary) {
105  pimpl->writePlyFile(file, imageSet, maxZ, binary);
106 }
107 
108 /******************** Implementation in pimpl class *******************/
109 
110 Reconstruct3D::Pimpl::Pimpl() {
111 }
112 
113 float* Reconstruct3D::Pimpl::createPointMap(const unsigned short* dispMap, int width,
114  int height, int rowStride, const float* q, unsigned short minDisparity) {
115 
116  // Allocate the buffer
117  if(pointMap.size() != static_cast<unsigned int>(4*width*height)) {
118  pointMap.resize(4*width*height);
119  }
120 
121 # ifdef __AVX2__
122  if(width % 16 == 0) {
123  return createPointMapAVX2(dispMap, width, height, rowStride, q, minDisparity);
124  } else
125 # endif
126 # ifdef __SSE2__
127  if(width % 8 == 0) {
128  return createPointMapSSE2(dispMap, width, height, rowStride, q, minDisparity);
129  } else
130 # endif
131  return createPointMapFallback(dispMap, width, height, rowStride, q, minDisparity);
132 }
133 
134 float* Reconstruct3D::Pimpl::createPointMap(const ImageSet& imageSet, unsigned short minDisparity) {
135  if(!imageSet.hasImageType(ImageSet::IMAGE_DISPARITY)) {
136  throw std::runtime_error("ImageSet does not contain a disparity map!");
137  }
138 
139  if(imageSet.getPixelFormat(ImageSet::IMAGE_DISPARITY) != ImageSet::FORMAT_12_BIT_MONO) {
140  throw std::runtime_error("Disparity map must have 12-bit pixel format!");
141  }
142 
143  return createPointMap(reinterpret_cast<unsigned short*>(imageSet.getPixelData(ImageSet::IMAGE_DISPARITY)), imageSet.getWidth(),
144  imageSet.getHeight(), imageSet.getRowStride(ImageSet::IMAGE_DISPARITY), imageSet.getQMatrix(), minDisparity);
145 }
146 
147 float* Reconstruct3D::Pimpl::createPointMapFallback(const unsigned short* dispMap, int width,
148  int height, int rowStride, const float* q, unsigned short minDisparity) {
149  // Code without SSE or AVX optimization
150  float* outputPtr = &pointMap[0];
151  int stride = rowStride / 2;
152 
153  for(int y = 0; y < height; y++) {
154  double qx = q[1]*y + q[3];
155  double qy = q[5]*y + q[7];
156  double qz = q[9]*y + q[11];
157  double qw = q[13]*y + q[15];
158 
159  for(int x = 0; x < width; x++) {
160  unsigned short intDisp = std::max(minDisparity, dispMap[y*stride + x]);
161  if(intDisp >= 0xFFF) {
162  intDisp = minDisparity; // Invalid disparity
163  }
164 
165  double d = intDisp / 16.0;
166  double w = qw + q[14]*d;
167 
168  *outputPtr = static_cast<float>((qx + q[2]*d)/w); // x
169  outputPtr++;
170 
171  *outputPtr = static_cast<float>((qy + q[6]*d)/w); // y
172  outputPtr++;
173 
174  *outputPtr = static_cast<float>((qz + q[10]*d)/w); // z
175  outputPtr+=2; // Consider padding
176 
177  qx += q[0];
178  qy += q[4];
179  qz += q[8];
180  qw += q[12];
181  }
182  }
183  return &pointMap[0];
184 }
185 
186 void Reconstruct3D::Pimpl::projectSinglePoint(int imageX, int imageY, unsigned short disparity,
187  const float* q, float& pointX, float& pointY, float& pointZ) {
188 
189  double d = disparity / 16.0;
190  double w = q[15] + q[14]*d;
191  pointX = static_cast<float>((imageX*q[0] + q[3])/w);
192  pointY = static_cast<float>((imageY*q[5] + q[7])/w);
193  pointZ = static_cast<float>(q[11]/w);
194 }
195 
196 # ifdef __AVX2__
197 float* Reconstruct3D::Pimpl::createPointMapAVX2(const unsigned short* dispMap, int width,
198  int height, int rowStride, const float* q, unsigned short minDisparity) {
199 
200  // Create column vectors of q
201  const __m256 qCol0 = _mm256_setr_ps(q[0], q[4], q[8], q[12], q[0], q[4], q[8], q[12]);
202  const __m256 qCol1 = _mm256_setr_ps(q[1], q[5], q[9], q[13], q[1], q[5], q[9], q[13]);
203  const __m256 qCol2 = _mm256_setr_ps(q[2], q[6], q[10], q[14], q[2], q[6], q[10], q[14]);
204  const __m256 qCol3 = _mm256_setr_ps(q[3], q[7], q[11], q[15], q[3], q[7], q[11], q[15]);
205 
206  // More constants that we need
207  const __m256i minDispVector = _mm256_set1_epi16(minDisparity);
208  const __m256i maxDispVector = _mm256_set1_epi16(0xFFF);
209  const __m256 scaleVector = _mm256_set1_ps(1.0/16.0);
210  const __m256i zeroVector = _mm256_set1_epi16(0);
211 
212  float* outputPtr = &pointMap[0];
213 
214  for(int y = 0; y < height; y++) {
215  const unsigned char* rowStart = &reinterpret_cast<const unsigned char*>(dispMap)[y*rowStride];
216  const unsigned char* rowEnd = &reinterpret_cast<const unsigned char*>(dispMap)[y*rowStride + 2*width];
217 
218  int x = 0;
219  for(const unsigned char* ptr = rowStart; ptr != rowEnd; ptr += 32) {
220  __m256i disparities = _mm256_load_si256(reinterpret_cast<const __m256i*>(ptr));
221 
222  // Find invalid disparities and set them to 0
223  __m256i validMask = _mm256_cmpgt_epi16(maxDispVector, disparities);
224  disparities = _mm256_and_si256(validMask, disparities);
225 
226  // Clamp to minimum disparity
227  disparities = _mm256_max_epi16(disparities, minDispVector);
228 
229  // Stupid AVX2 unpack mixes everything up! Lets swap the register beforehand.
230  __m256i disparitiesMixup = _mm256_permute4x64_epi64(disparities, 0xd8);
231 
232  // Convert to floats and scale with 1/16
233  __m256 floatDisp = _mm256_cvtepi32_ps(_mm256_unpacklo_epi16(disparitiesMixup, zeroVector));
234  __m256 dispScaled = _mm256_mul_ps(floatDisp, scaleVector);
235 
236  // Copy to array
237 #ifdef _MSC_VER
238  __declspec(align(32)) float dispArray[16];
239 #else
240  float dispArray[16]__attribute__((aligned(32)));
241 #endif
242  _mm256_store_ps(&dispArray[0], dispScaled);
243 
244  // Same for other half
245  floatDisp = _mm256_cvtepi32_ps(_mm256_unpackhi_epi16(disparitiesMixup, zeroVector));
246  dispScaled = _mm256_mul_ps(floatDisp, scaleVector);
247  _mm256_store_ps(&dispArray[8], dispScaled);
248 
249  // Iterate over disparities and perform matrix multiplication for each
250  for(int i=0; i<16; i+=2) {
251  // Create two vectors
252  __m256 vec = _mm256_setr_ps(x, y, dispArray[i], 1.0,
253  x+1, y, dispArray[i+1], 1.0);
254 
255  // Multiply with matrix
256  __m256 u1 = _mm256_shuffle_ps(vec,vec, _MM_SHUFFLE(0,0,0,0));
257  __m256 u2 = _mm256_shuffle_ps(vec,vec, _MM_SHUFFLE(1,1,1,1));
258  __m256 u3 = _mm256_shuffle_ps(vec,vec, _MM_SHUFFLE(2,2,2,2));
259  __m256 u4 = _mm256_shuffle_ps(vec,vec, _MM_SHUFFLE(3,3,3,3));
260 
261  __m256 prod1 = _mm256_mul_ps(u1, qCol0);
262  __m256 prod2 = _mm256_mul_ps(u2, qCol1);
263  __m256 prod3 = _mm256_mul_ps(u3, qCol2);
264  __m256 prod4 = _mm256_mul_ps(u4, qCol3);
265 
266  __m256 multResult = _mm256_add_ps(_mm256_add_ps(prod1, prod2), _mm256_add_ps(prod3, prod4));
267 
268  // Divide by w to receive point coordinates
269  __m256 point = _mm256_div_ps(multResult,
270  _mm256_shuffle_ps(multResult,multResult, _MM_SHUFFLE(3,3,3,3)));
271 
272  // Write result to memory
273  _mm256_store_ps(outputPtr, point);
274 
275  outputPtr += 8;
276  x+=2;
277  }
278  }
279  }
280 
281  return &pointMap[0];
282 }
283 #endif
284 
285 #ifdef __SSE2__
286 float* Reconstruct3D::Pimpl::createPointMapSSE2(const unsigned short* dispMap, int width,
287  int height, int rowStride, const float* q, unsigned short minDisparity) {
288 
289  // Create column vectors of q
290  const __m128 qCol0 = _mm_setr_ps(q[0], q[4], q[8], q[12]);
291  const __m128 qCol1 = _mm_setr_ps(q[1], q[5], q[9], q[13]);
292  const __m128 qCol2 = _mm_setr_ps(q[2], q[6], q[10], q[14]);
293  const __m128 qCol3 = _mm_setr_ps(q[3], q[7], q[11], q[15]);
294 
295  // More constants that we need
296  const __m128i minDispVector = _mm_set1_epi16(minDisparity);
297  const __m128i maxDispVector = _mm_set1_epi16(0xFFF);
298  const __m128 scaleVector = _mm_set1_ps(1.0/16.0);
299  const __m128i zeroVector = _mm_set1_epi16(0);
300 
301  float* outputPtr = &pointMap[0];
302 
303  for(int y = 0; y < height; y++) {
304  const unsigned char* rowStart = &reinterpret_cast<const unsigned char*>(dispMap)[y*rowStride];
305  const unsigned char* rowEnd = &reinterpret_cast<const unsigned char*>(dispMap)[y*rowStride + 2*width];
306 
307  int x = 0;
308  for(const unsigned char* ptr = rowStart; ptr != rowEnd; ptr += 16) {
309  __m128i disparities = _mm_load_si128(reinterpret_cast<const __m128i*>(ptr));
310 
311  // Find invalid disparities and set them to 0
312  __m128i validMask = _mm_cmplt_epi16(disparities, maxDispVector);
313  disparities = _mm_and_si128(validMask, disparities);
314 
315  // Clamp to minimum disparity
316  disparities = _mm_max_epi16(disparities, minDispVector);
317 
318  // Convert to floats and scale with 1/16
319  __m128 floatDisp = _mm_cvtepi32_ps(_mm_unpacklo_epi16(disparities, zeroVector));
320  __m128 dispScaled = _mm_mul_ps(floatDisp, scaleVector);
321 
322  // Copy to array
323 #ifdef _MSC_VER
324  __declspec(align(16)) float dispArray[8];
325 #else
326  float dispArray[8]__attribute__((aligned(16)));
327 #endif
328  _mm_store_ps(&dispArray[0], dispScaled);
329 
330  // Same for other half
331  floatDisp = _mm_cvtepi32_ps(_mm_unpackhi_epi16(disparities, zeroVector));
332  dispScaled = _mm_mul_ps(floatDisp, scaleVector);
333  _mm_store_ps(&dispArray[4], dispScaled);
334 
335  // Iterate over disparities and perform matrix multiplication for each
336  for(int i=0; i<8; i++) {
337  // Create vector
338  __m128 vec = _mm_setr_ps(static_cast<float>(x), static_cast<float>(y), dispArray[i], 1.0);
339 
340  // Multiply with matrix
341  __m128 u1 = _mm_shuffle_ps(vec,vec, _MM_SHUFFLE(0,0,0,0));
342  __m128 u2 = _mm_shuffle_ps(vec,vec, _MM_SHUFFLE(1,1,1,1));
343  __m128 u3 = _mm_shuffle_ps(vec,vec, _MM_SHUFFLE(2,2,2,2));
344  __m128 u4 = _mm_shuffle_ps(vec,vec, _MM_SHUFFLE(3,3,3,3));
345 
346  __m128 prod1 = _mm_mul_ps(u1, qCol0);
347  __m128 prod2 = _mm_mul_ps(u2, qCol1);
348  __m128 prod3 = _mm_mul_ps(u3, qCol2);
349  __m128 prod4 = _mm_mul_ps(u4, qCol3);
350 
351  __m128 multResult = _mm_add_ps(_mm_add_ps(prod1, prod2), _mm_add_ps(prod3, prod4));
352 
353  // Divide by w to receive point coordinates
354  __m128 point = _mm_div_ps(multResult,
355  _mm_shuffle_ps(multResult,multResult, _MM_SHUFFLE(3,3,3,3)));
356 
357  // Write result to memory
358  _mm_store_ps(outputPtr, point);
359 
360  outputPtr += 4;
361  x++;
362  }
363  }
364  }
365 
366  return &pointMap[0];
367 }
368 #endif
369 
370 void Reconstruct3D::Pimpl::writePlyFile(const char* file, const unsigned short* dispMap,
371  const unsigned char* image, int width, int height, ImageSet::ImageFormat format, int dispRowStride,
372  int imageRowStride, const float* q, double maxZ, bool binary) {
373 
374  float* pointMap = createPointMap(dispMap, width, height, dispRowStride, q, 0);
375 
376  // Count number of valid points
377  int pointsCount = 0;
378  if(maxZ >= 0) {
379  for(int i=0; i<width*height; i++) {
380  if(pointMap[4*i+2] <= maxZ) {
381  pointsCount++;
382  }
383  }
384  } else {
385  pointsCount = width*height;
386  }
387 
388  // Write file header
389  fstream strm(file, binary ? (ios::out | ios::binary) : ios::out);
390  strm << "ply" << endl;
391 
392  if(binary) {
393  strm << "format binary_little_endian 1.0" << endl;
394  } else {
395  strm << "format ascii 1.0" << endl;
396  }
397 
398  strm << "element vertex " << pointsCount << endl
399  << "property float x" << endl
400  << "property float y" << endl
401  << "property float z" << endl;
402  if (image != nullptr) {
403  // include RGB information only if a camera image was provided
404  strm << "property uchar red" << endl
405  << "property uchar green" << endl
406  << "property uchar blue" << endl;
407  }
408  strm << "end_header" << endl;
409 
410  // Write points
411  for(int i=0; i<width*height; i++) {
412  int y = i / width;
413  int x = i % width;
414 
415  if(maxZ < 0 || pointMap[4*i+2] <= maxZ) {
416  if(binary) {
417  // Write binary format
418  strm.write(reinterpret_cast<char*>(&pointMap[4*i]), sizeof(float)*3);
419  if (image == nullptr) {
420  // disparity only, no image data
421  } else if(format == ImageSet::FORMAT_8_BIT_RGB) {
422  const unsigned char* col = &image[y*imageRowStride + 3*x];
423  strm.write(reinterpret_cast<const char*>(col), 3*sizeof(*col));
424  } else if(format == ImageSet::FORMAT_8_BIT_MONO) {
425  const unsigned char* col = &image[y*imageRowStride + x];
426  unsigned char writeData[3] = {*col, *col, *col};
427  strm.write(reinterpret_cast<const char*>(writeData), sizeof(writeData));
428  } else if(format == ImageSet::FORMAT_12_BIT_MONO) {
429  const unsigned short* col = reinterpret_cast<const unsigned short*>(&image[y*imageRowStride + 2*x]);
430  unsigned char writeData[3] = {
431  (unsigned char)(*col >> 4),
432  (unsigned char)(*col >> 4),
433  (unsigned char)(*col >> 4)
434  };
435  strm.write(reinterpret_cast<const char*>(writeData), sizeof(writeData));
436  }
437  } else {
438  // Write ASCII format
439  if(std::isfinite(pointMap[4*i + 2])) {
440  strm << pointMap[4*i]
441  << " " << pointMap[4*i + 1]
442  << " " << pointMap[4*i + 2];
443  } else {
444  strm << "NaN NaN NaN";
445  }
446 
447  if (image == nullptr) {
448  // disparity only, no image data
449  strm << endl;
450  } else if(format == ImageSet::FORMAT_8_BIT_RGB) {
451  const unsigned char* col = &image[y*imageRowStride + 3*x];
452  strm << " " << static_cast<int>(col[0])
453  << " " << static_cast<int>(col[1])
454  << " " << static_cast<int>(col[2]) << endl;
455  } else if(format == ImageSet::FORMAT_8_BIT_MONO) {
456  const unsigned char* col = &image[y*imageRowStride + x];
457  strm << " " << static_cast<int>(*col)
458  << " " << static_cast<int>(*col)
459  << " " << static_cast<int>(*col) << endl;
460  } else if(format == ImageSet::FORMAT_12_BIT_MONO) {
461  const unsigned short* col = reinterpret_cast<const unsigned short*>(&image[y*imageRowStride + 2*x]);
462  strm << " " << static_cast<int>(*col >> 4)
463  << " " << static_cast<int>(*col >> 4)
464  << " " << static_cast<int>(*col >> 4) << endl;
465  }
466  }
467  }
468  }
469 }
470 
471 void Reconstruct3D::Pimpl::writePlyFile(const char* file, const ImageSet& imageSet,
472  double maxZ, bool binary) {
473  int indexDisp = imageSet.getIndexOf(ImageSet::IMAGE_DISPARITY);
474  int indexImg = imageSet.getIndexOf(ImageSet::IMAGE_LEFT);
475  if(indexDisp == -1) {
476  throw std::runtime_error("No disparity channel present, cannot create point map!");
477  }
478  if(imageSet.getPixelFormat(ImageSet::IMAGE_DISPARITY) != ImageSet::FORMAT_12_BIT_MONO) {
479  throw std::runtime_error("Disparity map must have 12-bit pixel format!");
480  }
481 
482  // write Ply file, passing image data for point colors, if available
483  writePlyFile(file, reinterpret_cast<unsigned short*>(imageSet.getPixelData(indexDisp)),
484  (indexImg == -1) ? nullptr : imageSet.getPixelData(indexImg),
485  imageSet.getWidth(), imageSet.getHeight(),
486  (indexImg == -1) ? ImageSet::FORMAT_8_BIT_MONO : imageSet.getPixelFormat(indexImg),
487  imageSet.getRowStride(indexDisp),
488  (indexImg == -1) ? 0 : imageSet.getRowStride(indexImg),
489  imageSet.getQMatrix(),
490  maxZ, binary);
491 }
492 
493 } // namespace
494 
int getRowStride(int imageNumber) const
Returns the row stride for the pixel data of one image.
Definition: imageset.h:216
const float * getQMatrix() const
Returns a pointer to the disparity-to-depth mapping matrix q.
Definition: imageset.h:291
int getIndexOf(ImageType what, bool throwIfNotFound=false) const
Returns the index of a specific image type.
Definition: imageset.cpp:208
int getHeight() const
Returns the height of each image.
Definition: imageset.h:205
ImageFormat
Image formats that can be transferred.
Definition: imageset.h:44
unsigned char * getPixelData(int imageNumber) const
Returns the pixel data for the given image.
Definition: imageset.h:270
void projectSinglePoint(int imageX, int imageY, unsigned short disparity, const float *q, float &pointX, float &pointY, float &pointZ)
Reconstructs the 3D location of one individual point.
bool hasImageType(ImageType what) const
Returns whether a left camera image is included in the enabled data.
Definition: imageset.h:430
A set of one to three images, but usually two (the left camera image and the disparity map)...
Definition: imageset.h:38
ImageFormat getPixelFormat(int imageNumber) const
Returns the pixel format for the given image.
Definition: imageset.h:243
float * createPointMap(const unsigned short *dispMap, int width, int height, int rowStride, const float *q, unsigned short minDisparity=1)
Reconstructs the 3D location of each pixel in the given disparity map.
void writePlyFile(const char *file, const unsigned short *dispMap, const unsigned char *image, int width, int height, ImageSet::ImageFormat format, int dispRowStride, int imageRowStride, const float *q, double maxZ=std::numeric_limits< double >::max(), bool binary=false)
Projects the given disparity map to 3D points and exports the result to a PLY file.
int getWidth() const
Returns the width of each image.
Definition: imageset.h:200
Nerian Vision Technologies