linkPhotoMosaic in Hardware

linkProblem Statement

Converting the image to a photo mosaic

linkBackground

linkSpatial Coherence

Spatial coherence refers to a defined phase relationship between different points in a cross section of a light beam. Let us consider 2 points p1 and p2 that lie in the same beam cross section (surface perpendicular to the direction of propagation). If the phase difference between the electric fields of both points remains constant at any moment, it is said that between both points there is a perfect spatial coherence.

For conventional light sources the coherence area is of the order of 0.0001 mm², while for lasers it is of the order of 1 mm².

The way to detect spatial coherence in a light beam is by Young's experiment.

linkProcessing the dataset

To make a scalable solution, the dataset is previously concatenated. The images are ordered by brightness, from lowest to the highest brightness. The implementation was developed in Python, code below:

1linkimport cv2

2linkfrom glob import glob

3linkfrom PIL import Image,ImageStat

4linkimport math

5link

6linkdef brightness( img_file ):

7link  im = Image.open(img_file)

8link  stat = ImageStat.Stat(im)

9link  r,g,b = stat.rms

10link  bright= math.sqrt(0.241*(r**2) + 0.691*(g**2) + 0.068*(b**2))

11link  return { 'brightness': bright, 'filename': img_file }

12link

13linkdef orderByBrightness(files):

14link dataset= []

15link for f in files:

16link   img_brightness = brightness(f)

17link   dataset.append(img_brightness)

18link return sorted(dataset, key = lambda i: i['brightness'])

19link

20linkfiles = glob("dataset/*.jpg")

21linkdataset = orderByBrightness(files)

22linkfirst = True

23linkimg_w = None

24linkfor i, data in enumerate(dataset):

25link if i< 52: continue

26link f = data['filename']

27link if first:

28link   first=False

29link   img_w = cv2.imread(f)

30link   continue

31link img = cv2.imread(f)

32link img_w = cv2.hconcat([img_w, img])

33link

34linkcv2.imwrite('concat_dataset.jpg', img_w)

linkFinal Results

linkImage

PhotomosaicHardware.js1linklet mosaic;

2linklet symbol1;

3linklet myImage;

4linklet debug;

5linklet slider;

6linkconst WIDTH_PIXEL = 64;

7linkconst HEIGHT_PIXEL = 64;

8linkconst NUM_IMAGES = 70;

9linkfunction preload() {

10link myImage = loadImage("/vc/docs/sketches/images/mandrill.png");

11link symbol1 = loadImage("/vc/docs/sketches/images/concat_dataset.jpg");

12link mosaic = loadShader(

13link   "/vc/docs/sketches/shaders/shader.vert",

14link   "/vc/docs/sketches/shaders/hardwarePhotomosaic.frag"

15link );

16link}

17link

18linkfunction setup() {

19link slider = createSlider(1, 6, 4,1);

20link slider.position(10, 10);

21link slider.style('width', '100px');

22link createCanvas(600, 600, WEBGL);

23link textureMode(NORMAL);

24link noStroke();

25link shader(mosaic);

26link mosaic.setUniform("image", myImage);

27link mosaic.setUniform("WIDTH_PIXEL", WIDTH_PIXEL);

28link mosaic.setUniform("NUM_IMAGES", NUM_IMAGES);

29link mosaic.setUniform("HEIGHT_PIXEL", HEIGHT_PIXEL);

30link debug = true;

31link mosaic.setUniform("debug", debug);

32link let img = symbol1;

33link mosaic.setUniform("symbol1", img);

34link}

35link

36linkfunction draw() {

37link mosaic.setUniform("resolution", Math.pow(10,slider.value()));

38link

39link background(33);

40link cover(true);

41link}

42link

43linkfunction cover(texture = false) {

44link beginShape();

45link if (texture) {

46link   vertex(-width / 2, -height / 2, 0, 0, 0);

47link   vertex(width / 2, -height / 2, 0, 1, 0);

48link   vertex(width / 2, height / 2, 0, 1, 1);

49link   vertex(-width / 2, height / 2, 0, 0, 1);

50link } else {

51link   vertex(-width / 2, -height / 2, 0);

52link   vertex(width / 2, -height / 2, 0);

53link   vertex(width / 2, height / 2, 0);

54link   vertex(-width / 2, height / 2, 0);

55link }

56link endShape(CLOSE);

57link}

58link

59linkfunction keyPressed() {

60link if (key === "d") {

61link   debug = !debug;

62link   mosaic.setUniform("debug", debug);

63link }

64link}

shader.vert1linkprecision highp float;

2linkattribute vec3 aPosition;

3linkattribute vec2 aTexCoord;

4linkattribute vec4 aVertexColor;

5linkuniform mat4 uProjectionMatrix;

6linkuniform mat4 uModelViewMatrix;

7linkvarying vec4 vVertexColor;

8linkvarying vec2 vTexCoord;

9link

10linkvoid main() {

11link vVertexColor = aVertexColor;

12link vTexCoord = aTexCoord;

13link gl_Position = uProjectionMatrix * uModelViewMatrix * vec4(aPosition, 1.0);

14link}

hardwarePhotomosaic.vert1linkprecision mediump float;

2linkuniform sampler2D image;

3linkuniform sampler2D symbol1;

4linkuniform bool debug;

5linkuniform float resolution;

6linkuniform float NUM_IMAGES;

7linkuniform float WIDTH_PIXEL;

8linkuniform float HEIGHT_PIXEL;

9linkvarying vec2 vTexCoord;

10linkvarying vec4 vVertexColor;

11link

12linkfloat module( float x , float y ){

13link   float flt_res = x-(y*(floor(x/y)));

14link   return flt_res;

15link}

16link

17linkvoid main() {

18link   vec2 symbolCoord=vTexCoord*resolution;

19link   vec2 imageCoord=floor(symbolCoord);

20link   symbolCoord=symbolCoord-imageCoord;

21link   imageCoord=imageCoord*vec2(1.0)/vec2(resolution);

22link   vec4 col=texture2D(image,imageCoord);

23link   float brightness = dot(col.xyz, vec3(0.2126, 0.7152, 0.0722));

24link   float temp=brightness*(NUM_IMAGES);

25link   float level=floor(temp);

26link   float scalingfactor = 1.0/NUM_IMAGES;

27link   float y0=0.0;

28link   float x0= module(level,NUM_IMAGES)*scalingfactor;

29link   vec2 myCoord=(symbolCoord*vec2(1.0)/vec2(NUM_IMAGES,1))+vec2(x0,y0);

30link   vec4 finalColor=texture2D(symbol1,myCoord);

31link   gl_FragColor = debug?finalColor:col;

32link}

linkVideo

There are two sliders that allow configuring the resolution. The First Slider decide the pow(1,100,1.000,10.000,100.000).The Second Slider is a number from 1 to 10. This configuration lets the users visualize multiple resolutions that generate a variety of results

PhotomosaicHardwareCamara.js1linklet mosaic;

2linklet symbol1;

3linklet myImage;

4linklet debug;

5linklet slider;

6linkconst WIDTH_PIXEL = 64;

7linkconst HEIGHT_PIXEL = 64;

8linkconst NUM_IMAGES = 70;

9linkfunction preload() {

10link myImage = createCapture(VIDEO);

11link myImage.hide();

12link symbol1 = loadImage("/vc/docs/sketches/images/concat_dataset.jpg");

13link mosaic = loadShader(

14link   "/vc/docs/sketches/shaders/shader.vert",

15link   "/vc/docs/sketches/shaders/hardwarePhotomosaic.frag"

16link );

17link}

18link

19linkfunction setup() {

20link slider = createSlider(1, 6, 4,1);

21link slider.position(10, 10);

22link slider.style('width', '100px');

23link createCanvas(600, 600, WEBGL);

24link textureMode(NORMAL);

25link noStroke();

26link shader(mosaic);

27link mosaic.setUniform("image", myImage);

28link mosaic.setUniform("WIDTH_PIXEL", WIDTH_PIXEL);

29link mosaic.setUniform("NUM_IMAGES", NUM_IMAGES);

30link mosaic.setUniform("HEIGHT_PIXEL", HEIGHT_PIXEL);

31link debug = true;

32link mosaic.setUniform("debug", debug);

33link let img = symbol1;

34link mosaic.setUniform("symbol1", img);

35link}

36link

37linkfunction draw() {

38link mosaic.setUniform("resolution", Math.pow(10,slider.value()));

39link

40link background(33);

41link cover(true);

42link}

43link

44linkfunction cover(texture = false) {

45link beginShape();

46link if (texture) {

47link   vertex(-width / 2, -height / 2, 0, 0, 0);

48link   vertex(width / 2, -height / 2, 0, 1, 0);

49link   vertex(width / 2, height / 2, 0, 1, 1);

50link   vertex(-width / 2, height / 2, 0, 0, 1);

51link } else {

52link   vertex(-width / 2, -height / 2, 0);

53link   vertex(width / 2, -height / 2, 0);

54link   vertex(width / 2, height / 2, 0);

55link   vertex(-width / 2, height / 2, 0);

56link }

57link endShape(CLOSE);

58link}

59link

60linkfunction keyPressed() {

61link if (key === "d") {

62link   debug = !debug;

63link   mosaic.setUniform("debug", debug);

64link }

65link}

shader.vert1linkprecision highp float;

2linkattribute vec3 aPosition;

3linkattribute vec2 aTexCoord;

4linkattribute vec4 aVertexColor;

5linkuniform mat4 uProjectionMatrix;

6linkuniform mat4 uModelViewMatrix;

7linkvarying vec4 vVertexColor;

8linkvarying vec2 vTexCoord;

9link

10linkvoid main() {

11link vVertexColor = aVertexColor;

12link vTexCoord = aTexCoord;

13link gl_Position = uProjectionMatrix * uModelViewMatrix * vec4(aPosition, 1.0);

14link}

hardwarePhotomosaic.vert1linkprecision mediump float;

2linkuniform sampler2D image;

3linkuniform sampler2D symbol1;

4linkuniform bool debug;

5linkuniform float resolution;

6linkuniform float NUM_IMAGES;

7linkuniform float WIDTH_PIXEL;

8linkuniform float HEIGHT_PIXEL;

9linkvarying vec2 vTexCoord;

10linkvarying vec4 vVertexColor;

11link

12linkfloat module( float x , float y ){

13link   float flt_res = x-(y*(floor(x/y)));

14link   return flt_res;

15link}

16link

17linkvoid main() {

18link   vec2 symbolCoord=vTexCoord*resolution;

19link   vec2 imageCoord=floor(symbolCoord);

20link   symbolCoord=symbolCoord-imageCoord;

21link   imageCoord=imageCoord*vec2(1.0)/vec2(resolution);

22link   vec4 col=texture2D(image,imageCoord);

23link   float brightness = dot(col.xyz, vec3(0.2126, 0.7152, 0.0722));

24link   float temp=brightness*(NUM_IMAGES);

25link   float level=floor(temp);

26link   float scalingfactor = 1.0/NUM_IMAGES;

27link   float y0=0.0;

28link   float x0= module(level,NUM_IMAGES)*scalingfactor;

29link   vec2 myCoord=(symbolCoord*vec2(1.0)/vec2(NUM_IMAGES,1))+vec2(x0,y0);

30link   vec4 finalColor=texture2D(symbol1,myCoord);

31link   gl_FragColor = debug?finalColor:col;

32link}