vc | Convolutional Mask

linkBackground

convolution is a mathematical operation on two functions (f and g) that produces a third function (f*g) that expresses how the shape of one is modified by the other. in the case of image processing the convolution is the proccess of adding each element of the image to its local neighbors , weighted by the kernel to extract certain features from an input image.The kernels will define the size of the convolution, the weights applied to it, and an anchor point usually positioned at the center.The origin is the position of the kernel which is above (conceptually) the current output pixel. For a symmetric kernel, the origin is usually the center element.

Of course we are not restricted to 3x3 kernels - this was only done for simplicity. Kernels can be of just about any size. More sophisticated kernels are typically larger, in fact many image processing software packages have options to customize a kernel.

One of the earliest uses of the convolution integral appeared in D'Alembert's derivation of Taylor's theorem in Recherches sur différents points importants du système du monde, published in 1754. Soon thereafter, convolution operations appear in the works of Pierre Simon Laplace, Jean-Baptiste Joseph Fourier, Siméon Denis Poisson, and others. The term itself did not come into wide use until the 1950s or 60s.

linkCode and Results

For this, we use a vertex shader and a fragment shader (The same for image, video and camera recording) and just change the Javascript file.

The kernel matriz is a 3*3 matriz, and it is passed to the shaders in a uniform vector of floats.

The results and code can be seed above.

linkImage

ConvolutionImage.js1linklet theShader;

2linklet img;

3linklet reproduce = false;

4link

5linkconst Blur_Kernel= [ 0.11, 0.11, 0.11 ,0.11, 0.11, 0.11, 0.11, 0.11, 0.11]; 

6linklet Border_Detection= [ -1.0, -1.0, -1.0 , -1.0,  8.0, -1.0 , -1.0, -1.0, -1.0  ];

7linkconst Emboss= [ 1,  1,  0, 1,  0, -1 , 0,  -1,  -1]; 

8linkconst Sharpe= [ 0, -1, 0 , -1,  5, -1, 0, -1, 0 ]; 

9link

10linklet contador=0;

11linklet matrixCarrousel= [Blur_Kernel,Border_Detection,Emboss,Sharpe];

12linklet kernel = matrixCarrousel[0] ;

13link

14linkfunction preload() {

15link theShader = loadShader('/vc/docs/sketches/workshop2/exercice2/shader.vert', '/vc/docs/sketches/workshop2/exercice2/edge.frag');

16link img = loadImage('/vc/docs/sketches/lenna.png');

17link}

18link

19linkfunction setup() {

20link createCanvas(640, 400, WEBGL);

21link noStroke();

22link textureMode(NORMAL); 

23link shader(theShader);

24link theShader.setUniform('texture', img);

25link theShader.setUniform('texOffset',[1/img.width,1/img.height]);

26link button=createButton('Change Kernel!');

27link button.position(300,350);

28link button.mousePressed(changeMatrix);

29link}

30linkfunction draw() {

31link background(0);

32link

33link beginShape() 

34link vertex(-width / 2, height / 2, 0, 0, 1);

35link vertex(width / 2, height / 2, 0, 1, 1);

36link vertex(width / 2, -height / 2, 0, 1, 0);

37link vertex(-width / 2, -height / 2, 0, 0, 0);

38link theShader.setUniform('kernel', kernel);

39link endShape(CLOSE)

40link

41link}

42linkfunction changeMatrix(){

43link contador=(contador+1)%matrixCarrousel.length;

44link kernel=matrixCarrousel[contador];

45link}

ConvolutionImage.frag1linkprecision mediump float;

2linkuniform float kernel[9];

3link

4linkvec4 col[9];

5linkvec2 tc[9];

6link

7link// texture is sent by the sketch

8linkuniform sampler2D texture;

9linkuniform vec2 texOffset;

10link

11link// interpolated color (same name and type as in vertex shader)

12linkvarying vec4 vVertexColor;

13link

14link// interpolated texcoord (same name and type as in vertex shader)

15linkvarying vec2 vTexCoord;

16link

17linkconst vec4 lumcoeff = vec4(0.299, 0.587, 0.114, 0);

18link

19linkvoid main() {

20link // texture2D(texture, vTexCoord) samples texture at vTexCoord 

21link // and returns the normalized texel color

22link // texel color times vVertexColor gives the final normalized pixel col[o]r

23link tc[0] = vTexCoord + vec2(-texOffset.s, -texOffset.t);

24link tc[1] = vTexCoord + vec2(         0.0, -texOffset.t);

25link tc[2] = vTexCoord + vec2(+texOffset.s, -texOffset.t);

26link tc[3] = vTexCoord + vec2(-texOffset.s,          0.0);

27link tc[4] = vTexCoord + vec2(         0.0,          0.0);

28link tc[5] = vTexCoord + vec2(+texOffset.s,          0.0);

29link tc[6] = vTexCoord + vec2(-texOffset.s, +texOffset.t);

30link tc[7] = vTexCoord + vec2(         0.0, +texOffset.t);

31link tc[8] = vTexCoord + vec2(+texOffset.s, +texOffset.t);

32link

33link for ( int i=0;i<9;i++){

34link   col[i] = texture2D(texture, tc[i]);

35link }

36link

37link vec4 sum = kernel[0] * col[0];

38link for (int i = 1; i<9 ; i++){

39link   vec4 pc =  kernel[i] * col[i];

40link   sum += pc;

41link }

42link gl_FragColor = vec4(vec3(sum), 1.0) * vVertexColor;

43link

44link}

linkVideo

ConvolutionVideo.js1linklet theShader;

2linklet fingers;

3linklet reproduce = false;

4link

5linkconst Blur_Kernel= [ 0.11, 0.11, 0.11 ,0.11, 0.11, 0.11, 0.11, 0.11, 0.11]; 

6linklet Border_Detection= [ -1.0, -1.0, -1.0 , -1.0,  8.0, -1.0 , -1.0, -1.0, -1.0  ];

7linkconst Emboss= [ 1,  1,  0, 1,  0, -1 , 0,  -1,  -1]; 

8linkconst Sharpe= [ 0, -1, 0 , -1,  5, -1, 0, -1, 0 ]; 

9link

10linklet contador=0;

11linklet matrixCarrousel= [Blur_Kernel,Border_Detection,Emboss,Sharpe];

12linklet kernel = matrixCarrousel[0] ;

13link

14linkfunction preload() {

15link theShader = loadShader('/vc/docs/sketches/workshop2/exercice2/shader.vert', '/vc/docs/sketches/workshop2/exercice2/edge.frag');

16link fingers = createVideo(['/vc/docs/sketches/fingers.mov', '/vc/docs/sketches/fingers.webm']);

17link fingers.hide();

18link}

19link

20linkfunction setup() {

21link createCanvas(640, 400, WEBGL);

22link noStroke();

23link textureMode(NORMAL); 

24link shader(theShader);

25link theShader.setUniform('texture', fingers);

26link theShader.setUniform('texOffset',[1/fingers.width,1/fingers.height]);

27link button=createButton('Change Kernel!');

28link button.position(300,350);

29link button.mousePressed(changeMatrix);

30link}

31linkfunction draw() {

32link background(0);

33link

34link beginShape() 

35link vertex(-width / 2, height / 2, 0, 0, 1);

36link vertex(width / 2, height / 2, 0, 1, 1);

37link vertex(width / 2, -height / 2, 0, 1, 0);

38link vertex(-width / 2, -height / 2, 0, 0, 0);

39link theShader.setUniform('kernel', kernel);

40link endShape(CLOSE)

41link

42link}

43linkfunction changeMatrix(){

44link contador=(contador+1)%matrixCarrousel.length;

45link kernel=matrixCarrousel[contador];

46link}

linkCamera

ConvolutionCamara.js1linklet theShader;

2linklet fingers;

3linklet reproduce = false;

4link

5linkconst Blur_Kernel= [ 0.11, 0.11, 0.11 ,0.11, 0.11, 0.11, 0.11, 0.11, 0.11]; 

6linklet Border_Detection= [ -1.0, -1.0, -1.0 , -1.0,  8.0, -1.0 , -1.0, -1.0, -1.0  ];

7linkconst Emboss= [ 1,  1,  0, 1,  0, -1 , 0,  -1,  -1]; 

8linkconst Sharpe= [ 0, -1, 0 , -1,  5, -1, 0, -1, 0 ]; 

9link

10linklet contador=0;

11linklet matrixCarrousel= [Blur_Kernel,Border_Detection,Emboss,Sharpe];

12linklet kernel = matrixCarrousel[0] ;

13link

14linkfunction preload() {

15link theShader = loadShader('/vc/docs/sketches/workshop2/exercice2/shader.vert', '/vc/docs/sketches/workshop2/exercice2/edge.frag');

16link fingers = createCapture(VIDEO)

17link fingers.hide();

18link}

19link

20linkfunction setup() {

21link createCanvas(640, 400, WEBGL);

22link noStroke();

23link textureMode(NORMAL); 

24link shader(theShader);

25link theShader.setUniform('texture', fingers);

26link theShader.setUniform('texOffset',[1/fingers.width,1/fingers.height]);

27link button=createButton('Change Kernel!');

28link button.position(300,350);

29link button.mousePressed(changeMatrix);

30link}

31linkfunction draw() {

32link background(0);

33link

34link beginShape() 

35link vertex(-width / 2, height / 2, 0, 0, 1);

36link vertex(width / 2, height / 2, 0, 1, 1);

37link vertex(width / 2, -height / 2, 0, 1, 0);

38link vertex(-width / 2, -height / 2, 0, 0, 0);

39link theShader.setUniform('kernel', kernel);

40link endShape(CLOSE)

41link

42link}

43linkfunction changeMatrix(){

44link contador=(contador+1)%matrixCarrousel.length;

45link kernel=matrixCarrousel[contador];

46link}