Rectifying with a Shader ExampleΒΆ

To rectify an image with a shader, you can upload both the image data and the distortion maps and apply them as textures to an object such as a quad. Instead of sampling the image data directly, your fragment shader samples the distortion map texture. You then use the texture coordinates from the distortion texture to look up the brightness value in the image texture. See Rectifying with a Shader for more information about the distortion map texture and the fragment shader to use it.

This example builds on Textured Quad Example. Instead of texturing the quad with the uncorrected image data, this example rectifies the image data to a second texture buffer and uses the result. You will notice that the image no longer fills the entire rectangular area. This is because rays of light that would have filled in the border areas were actually outside the field of view of the camera.

The main changes to this example occur in the display() function. For each pixel in the target texture, the function generates the slopes of the camera ray intersecting that point and looks up the correct brightness value in the uncorrected image buffer (as described in Rectifying Image Points). The result is the a texture containing the rectified image.

 1 #undef __cplusplus
  2
  3 #include <stdio.h>
  4 #include <stdlib.h>
  5
  6 #ifdef _WIN32
  7 #include <Windows.h>
  8 #else
  9 #include <unistd.h>
 10 #endif
 11
 12 #include <time.h>
 13 #include "LeapC.h"
 14 #include "ExampleConnection.h"
 15 #include "GLutils.h"
 16 #include <math.h>
 17 #include <string.h>
 18
 19 static const char* GLSL_VERT = STRINGIFY(
 20     \n#version 120\n
 21
 22     void main()
 23     {
 24         gl_Position = ftransform();
 25         gl_TexCoord[0]=gl_MultiTexCoord0;
 26     }
 27 );
 28 static const char* GLSL_FRAG = STRINGIFY(
 29     \n#version 120\n
 30     uniform sampler2D rawTexture;
 31     uniform sampler2D distortionTexture;
 32
 33     void main()
 34     {
 35         vec2 distortionIndex = texture2D(distortionTexture, gl_TexCoord[0].st).xy;
 36         float hIndex = distortionIndex.r;
 37         float vIndex = distortionIndex.g;
 38
 39         if(vIndex > 0.0 && vIndex < 1.0 && hIndex > 0.0 && hIndex < 1.0)
 40         {
 41             gl_FragColor = vec4(texture2D(rawTexture, distortionIndex).rrr, 1.0);
 42         }
 43         else
 44         {
 45             gl_FragColor = vec4(0.2, 0.0, 0.0, 1.0);
 46         }
 47     }
 48 );
 49
 50 GLsizei image_width;
 51 GLsizei image_height;
 52 void* image_buffer = NULL;
 53 float* distortion_buffer_left = NULL;
 54 float* distortion_buffer_right = NULL;
 55 uint64_t image_size = 0;
 56 bool imageReady = false;
 57 uint64_t currentDistortionId = 0;
 58
 59 int window; // GLUT window handle
 60 int windowWidth = 800;
 61 int windowHeight = 400;
 62
 63 GLuint vertShader, fragShader, program;
 64 GLuint rawTextureLeft = 0;
 65 GLuint rawTextureRight = 0;
 66 GLuint distortionTextureLeft = 0;
 67 GLuint distortionTextureRight = 0;
 68
 69 /** Callback for when an image is available. */
 70 static void OnImage(const LEAP_IMAGE_EVENT *imageEvent){
 71   const LEAP_IMAGE_PROPERTIES* properties = &imageEvent->image[0].properties;
 72   if (properties->bpp != 1)
 73     return;
 74
 75   if (properties->width*properties->height != image_size) {
 76     void* prev_image_buffer = image_buffer;
 77     image_width = properties->width;
 78     image_height = properties->height;
 79     image_size = image_width*image_height;
 80     image_buffer = malloc(2 * image_size);
 81     if (prev_image_buffer)
 82       free(prev_image_buffer);
 83   }
 84
 85   memcpy(image_buffer, (char*)imageEvent->image[0].data + imageEvent->image[0].offset, image_size);
 86   memcpy((char*)image_buffer + image_size, (char*)imageEvent->image[1].data + imageEvent->image[1].offset, image_size);
 87
 88   //Save the distortion data if it's version id changes
 89   if(currentDistortionId != imageEvent->image[0].matrix_version){
 90     size_t distortion_size = 64 * 64 * 2;
 91     distortion_buffer_left = malloc(sizeof(float) * distortion_size);
 92     distortion_buffer_right = malloc(sizeof(float) * distortion_size);
 93     memcpy(distortion_buffer_left, imageEvent->image[0].distortion_matrix, sizeof(float) * distortion_size);
 94     memcpy(distortion_buffer_right, imageEvent->image[1].distortion_matrix, sizeof(float) * distortion_size);
 95     currentDistortionId = imageEvent->image[0].matrix_version;
 96   }
 97   imageReady = true;
 98 }
 99
100 static void display(void)
101 {
102   glMatrixMode(GL_MODELVIEW);
103   glLoadIdentity();
104
105   glPushMatrix();
106   glClearColor(1.0f, 1.0f, 1.0f, 1.0f );
107   glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
108
109   if(imageReady){
110     //Setup perspective and view matrices
111     glMatrixMode(GL_PROJECTION);
112     checkGLError("set matrix mode");
113     glLoadIdentity();
114     setPerspectiveFrustrum(96, windowWidth/windowHeight, 1, 20);
115     checkGLError("set frustrum");
116
117     glMatrixMode(GL_MODELVIEW);
118     glPushMatrix();
119     glTranslatef(1.05f, 0.0, -1.01f);
120
121     glEnable(GL_TEXTURE_2D);
122     glUseProgram(program);
123
124     // Left camera image
125     glActiveTexture(GL_TEXTURE0);
126     glBindTexture(GL_TEXTURE_2D, rawTextureLeft);
127     glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, image_width, image_height, 0, GL_RED, GL_UNSIGNED_BYTE, image_buffer);
128     checkGLError("Initializing raw texture");
129
130     glActiveTexture(GL_TEXTURE1);
131     glBindTexture(GL_TEXTURE_2D, distortionTextureLeft);
132     glTexImage2D(GL_TEXTURE_2D, 0, GL_RG, 64, 64, 0, GL_RG, GL_FLOAT, distortion_buffer_left);
133     checkGLError("Initializing distortion texture");
134
135     glBegin(GL_QUADS); // Draw A Quad for camera image
136         glTexCoord2f(0.0f, 1.0f); glVertex3f(-1.0f, 1.0f, 0.0f); // Top Left
137         glTexCoord2f(1.0f, 1.0f); glVertex3f( 1.0f, 1.0f, 0.0f); // Top Right
138         glTexCoord2f(1.0f, 0.0f); glVertex3f( 1.0f, -1.0f, 0.0f); // Bottom Right
139         glTexCoord2f(0.0f, 0.0f); glVertex3f(-1.0f, -1.0f, 0.0f); // Bottom Left
140     glEnd();
141
142     // Right camera image
143     glActiveTexture(GL_TEXTURE0);
144     glBindTexture(GL_TEXTURE_2D, rawTextureRight);
145     glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, image_width, image_height, 0, GL_RED, GL_UNSIGNED_BYTE,
146       (GLvoid*)((size_t)(image_buffer) + (image_width * image_height)));
147     checkGLError("Updating raw texture with right image");
148
149     glActiveTexture(GL_TEXTURE1);
150     glBindTexture(GL_TEXTURE_2D, distortionTextureRight);
151     glTexImage2D(GL_TEXTURE_2D, 0, GL_RG, 64, 64, 0, GL_RG, GL_FLOAT, distortion_buffer_right);
152     checkGLError("Updating distortion texture with right distortion map");
153
154     glTranslatef(-2.01f, 0.0, 0.0);
155     glBegin(GL_QUADS);
156         glTexCoord2f(0.0f, 1.0f); glVertex3f(-1.0f, 1.0f, 0.0f); // Top Left
157         glTexCoord2f(1.0f, 1.0f); glVertex3f( 1.0f, 1.0f, 0.0f); // Top Right
158         glTexCoord2f(1.0f, 0.0f); glVertex3f( 1.0f,-1.0f, 0.0f); // Bottom Right
159         glTexCoord2f(0.0f, 0.0f); glVertex3f(-1.0f,-1.0f, 0.0f); // Bottom Left
160     glEnd();
161
162     glPopMatrix();
163
164     imageReady = false;
165   }
166   glFlush();
167   glPopMatrix();
168   glutSwapBuffers();
169 }
170
171 static void reshape(int w, int h)
172 {
173   glViewport(0, 0, (GLsizei) w, (GLsizei) h);
174 }
175
176 static void keyboard(unsigned char key, int x, int y)
177 {
178   switch((char)key) {
179   case 'q':
180   case 27:  // ESC
181     glutDestroyWindow(window);
182     CloseConnection();
183     if(image_buffer) free(image_buffer);
184     exit(0);
185   default:
186     break;
187   }
188 }
189
190 static void idle(void){
191   if(imageReady)
192     glutPostRedisplay();
193 }
194
195 int main(int argc, char *argv[])
196 {
197   ConnectionCallbacks.on_image        = OnImage;
198
199   LEAP_CONNECTION *connection = OpenConnection();
200   LeapSetPolicyFlags(*connection, eLeapPolicyFlag_Images, 0);
201
202   while(!IsConnected){
203     millisleep(250);
204   }
205
206   glutInit(&argc, argv);
207   glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGBA);
208   glutInitWindowSize(windowWidth, windowHeight);
209   window = glutCreateWindow("LeapC Distortion Shader Example");
210
211 #if defined(GLEW_VERSION)
212   GLenum err = glewInit();
213   if (err != GLEW_OK) {
214     /* Problem: glewInit failed, something is seriously wrong. */
215     fprintf(stderr, "Error: %s\n", glewGetErrorString(err));
216     return 1;
217   }
218 #endif
219
220   // GLUT callbacks
221   glutIdleFunc(idle);
222   glutReshapeFunc(reshape);
223   glutKeyboardFunc(keyboard);
224   glutDisplayFunc(display);
225
226   // init GL
227   glClearColor(0.0, 0.0, 0.0, 0.0);
228   glColor3f(1.0, 1.0, 1.0);
229
230   // init shader and textures
231
232   //Create the shader program
233   program = createProgram(GLSL_VERT, GLSL_FRAG);
234   glUseProgram(program);
235   GLuint rawSampler = glGetUniformLocation(program, "rawTexture");
236   GLuint distortionSampler  = glGetUniformLocation(program, "distortionTexture");
237   glUniform1i(rawSampler, 0);
238   glUniform1i(distortionSampler,  1);
239
240   //Create textures
241   rawTextureLeft = createTextureReference();
242   distortionTextureLeft = createTextureReference();
243   rawTextureRight = createTextureReference();
244   distortionTextureRight = createTextureReference();
245
246   // Start GLUT loop
247   glutMainLoop();
248
249   CloseConnection();
250   free(distortion_buffer_left);
251   free(distortion_buffer_right);
252   return 0;
253 }
254 //End-of-Sample

This example is only supported on platforms for which a working version of GLUT exists. However, it should not be overly difficult to port the example to a different OpenGL-based context.


Back to top