File-Input working with Source & PixelCaputure

src/input/PixelCapture.js

@@ -0,0 +1,120 @@
+import {
+    computeGray
+} from '../common/cv_utils';
+import {sleep} from '../common/utils';
+
+function adjustCanvasSize(input, canvas) {
+    if (input instanceof HTMLVideoElement) {
+        if (canvas.height !== input.videoHeight || canvas.width !== input.videoWidth) {
+            console.log('adjusting canvas size', input.videoHeight, input.videoWidth);
+            canvas.height = input.videoHeight;
+            canvas.width = input.videoWidth;
+            return true;
+        }
+        return false;
+    } else if (typeof input.width !== 'undefined') {
+        if (canvas.height !== input.height || canvas.width !== input.width) {
+            console.log('adjusting canvas size', input.height, input.width);
+            canvas.height = input.height;
+            canvas.width = input.width;
+            return true;
+        }
+        return false;
+    } else {
+        throw new Error('Not a video element!');
+    }
+}
+
+function getViewPort(target) {
+    if (target && target.nodeName && target.nodeType === 1) {
+        return target;
+    } else {
+        // Use '#interactive.viewport' as a fallback selector (backwards compatibility)
+        var selector = typeof target === 'string' ? target : '#interactive.viewport';
+        return document.querySelector(selector);
+    }
+}
+
+function getOrCreateCanvas(source, target) {
+    const $viewport = getViewPort(target);
+    let $canvas = $viewport.querySelector("canvas.imgBuffer");
+    if (!$canvas) {
+        $canvas = document.createElement("canvas");
+        $canvas.className = "imgBuffer";
+        if ($viewport && source.type === "IMAGE") {
+            $viewport.appendChild($canvas);
+        }
+    }
+    return $canvas;
+}
+
+export function fromSource(source, {target = "#interactive.viewport"} = {}) {
+    var drawable = source.getDrawable();
+    var $canvas = null;
+    var ctx = null;
+    var bytePool = [];
+
+    if (drawable instanceof HTMLVideoElement
+          || drawable instanceof HTMLImageElement) {
+        $canvas = getOrCreateCanvas(source, target);
+        ctx = $canvas.getContext('2d');
+    }
+
+    if (drawable instanceof HTMLCanvasElement) {
+        $canvas = drawable;
+        ctx = drawable.getContext('2d');
+    }
+
+    function nextAvailableBuffer() {
+        var i;
+        var buffer;
+        var bytesRequired = ($canvas.height * $canvas.width);
+        for (i = 0; i < bytePool.length; i++) {
+            buffer = bytePool[i];
+            if (buffer && buffer.buffer.byteLength === bytesRequired) {
+                return bytePool[i];
+            }
+        }
+        buffer = new Uint8Array(bytesRequired);
+        bytePool.push(buffer);
+        console.log("Added new entry to bufferPool", bytesRequired);
+        return buffer;
+    }
+
+    return {
+        grabFrameData: function grabFrameData() {
+            const {viewport, canvas: canvasSize} = source.getDimensions();
+            const sx = viewport.x;
+            const sy = viewport.y;
+            const sWidth = viewport.width;
+            const sHeight = viewport.height;
+            const dx = 0;
+            const dy = 0;
+            const dWidth = canvasSize.width;
+            const dHeight = canvasSize.height;
+
+            adjustCanvasSize(canvasSize, $canvas);
+            if ($canvas.height < 10 || $canvas.width < 10) {
+                console.log('$canvas not initialized. Waiting 100ms and then continuing');
+                return sleep(100).then(grabFrameData);
+            }
+            if (!(drawable instanceof HTMLCanvasElement)) {
+                ctx.drawImage(drawable, sx, sy, sWidth, sHeight, dx, dy, dWidth, dHeight);
+            }
+            var imageData = ctx.getImageData(0, 0, $canvas.width, $canvas.height).data;
+            var buffer = nextAvailableBuffer();
+            computeGray(imageData, buffer);
+            return Promise.resolve({
+                width: $canvas.width,
+                height: $canvas.height,
+                data: buffer,
+            });
+        },
+        getSource: function() {
+            return source;
+        },
+        getCanvas: function() {
+            return $canvas;
+        },
+    };
+}

src/input/Source.js

@@ -0,0 +1,180 @@
+import {determineOrientation, PORTRAIT} from '../common/device';
+import CameraAccess from './camera_access';
+
+export function fromCamera(constraints) {
+    if (!constraints) {
+        constraints = {width: {ideal: 540}, height: {ideal: 540}, aspectRatio: {ideal: 1}, facingMode: 'environment'};
+    }
+    var orientation = determineOrientation();
+    var videoConstraints = constraints;
+    if (orientation === PORTRAIT) {
+        constraints = Object.assign({}, constraints, {
+            width: constraints.height,
+            height: constraints.width,
+        });
+    }
+
+    const video = document.querySelector('video');
+    CameraAccess.request(videoConstraints, video)
+    .then(function(mediastream) {
+        const track = mediastream.getVideoTracks()[0];
+        return {
+            type: "CAMERA",
+            getDimensions() {
+                const viewport = {
+                    x: 0,
+                    y: 0,
+                    width: video.videoWidth,
+                    height: video.videoHeight,
+                };
+
+                if (constraints.zoom && constraints.zoom.exact > 1) {
+                    const zoom = constraints.zoom.exact;
+                    viewport.width = Math.floor(video.videoWidth / zoom);
+                    viewport.height = Math.floor(video.videoHeight / zoom);
+                    viewport.x = Math.floor((video.videoWidth - viewport.width) / 2);
+                    viewport.y = Math.floor((video.videoHeight - viewport.height) / 2);
+                }
+
+                return {
+                    viewport,
+                    canvas: {
+                        width: constraints.width,     // AR
+                        height: constraints.height,   // AR
+                    },
+                };
+            },
+            getConstraints: function() {
+                return videoConstraints;
+            },
+            getDrawable: function() {
+                return video;
+            },
+            applyConstraints: function(constraints) {
+                track.stop();
+                videoConstraints = Object.assign({}, constraints);
+                if (determineOrientation() === PORTRAIT) {
+                    constraints = Object.assign({}, constraints, {
+                        width: constraints.height,
+                        height: constraints.width,
+                    });
+                }
+                console.log(videoConstraints, constraints);
+                if (constraints.zoom && constraints.zoom.exact > 1) {
+                    constraints.width.ideal = Math.floor(constraints.width.ideal * constraints.zoom.exact);
+                    constraints.height.ideal = Math.floor(constraints.height.ideal * constraints.zoom.exact);
+                    delete constraints.zoom;
+                }
+                return CameraAccess.request(videoConstraints, video);
+            },
+            getLabel: function() {
+                return track.label;
+            }
+        };
+    });
+}
+
+export function fromCanvas(input) {
+    var $canvas = null;
+    if (typeof input === 'string') {
+        $canvas = document.querySelector(input);
+    } else if (input instanceof HTMLCanvasElement) {
+        $canvas = input;
+    } else {
+        return Promise.reject("fromCanvas needs a selector or HTMLCanvasElement");
+    }
+
+    return Promise.resolve({
+        type: "CANVAS",
+        getWidth: function() {
+            return $canvas.width;
+        },
+        getHeight: function() {
+            return $canvas.height;
+        },
+        getDrawable: function() {
+            return $canvas;
+        },
+        getLabel: function() {
+            return $canvas.getAttribute('id');
+        },
+        getConstraints: function() {
+            return {};
+        },
+        applyConstraints: function() {
+            console.log('CanvasSource.applyConstraints not implemented');
+        }
+    });
+}
+
+export function fromImage(input, constraints = {width: 800, height: 800}) {
+    var $image = null;
+    var src = null;
+    if (typeof input === 'string') {
+        // data or url, or queryString
+        $image = new Image();
+        src = input;
+    } else if (input instanceof HTMLImageElement) {
+        $image = input;
+    } else if (input instanceof File) {
+        $image = new Image();
+        src = URL.createObjectURL(input);
+    } else {
+        return Promise.reject("fromImage needs a src, HTMLImageElement or File");
+    }
+    return new Promise(function(resolve, reject) {
+        if (src || !$image.complete) {
+            console.log('Adding eventlistener');
+            $image.addEventListener('load', function() {
+                resolve();
+            }, false);
+            $image.addEventListener('error', function(e) {
+                reject(e);
+            }, false);
+            if (src) {
+                console.log(`Setting src = ${src}`);
+                $image.src = src;
+            }
+        } else {
+            return resolve();
+        }
+    })
+    .then(() => {
+        const width = $image.naturalWidth;
+        const height = $image.naturalHeight;
+        const imageAR = width / height;
+
+        const calculatedWidth = imageAR > 1 ? constraints.width : Math.floor((imageAR) * constraints.width);
+        const calculatedHeight = imageAR > 1 ? Math.floor((1 / imageAR) * constraints.width) : constraints.width;
+
+        return {
+            type: "IMAGE",
+            getDimensions() {
+                return {
+                    viewport: {
+                        width: $image.naturalWidth,   // AR
+                        height: $image.naturalHeight, // AR
+                        x: 0,                         // AR
+                        y: 0,                         // AR
+                    },
+                    canvas: {
+                        width: calculatedWidth,     // AR
+                        height: calculatedHeight,   // AR
+                    },
+                };
+            },
+            getDrawable: function() {
+                return $image;
+            },
+            getLabel: function() {
+                return $image.src;
+            },
+            getConstraints: function() {
+                return constraints;
+            },
+            applyConstraints: function() {
+                console.log('ImageSource.applyConstraints not implemented');
+            }
+        };
+    });
+}

src/input/camera_access.js

@@ -8,7 +8,7 @@ const facingMatching = {
 
 var streamRef;
 
-function waitForVideo(video) {
+function waitForVideo(video, stream) {
     return new Promise((resolve, reject) => {
         let attempts = 10;
 
@@ -18,7 +18,7 @@ function waitForVideo(video) {
                     if (ENV.development) {
                         console.log(video.videoWidth + "px x " + video.videoHeight + "px");
                     }
-                    resolve();
+                    resolve(stream);
                 } else {
                     window.setTimeout(checkVideo, 500);
                 }
@@ -46,7 +46,7 @@ function initCamera(video, constraints) {
             video.srcObject = stream;
             video.addEventListener('loadedmetadata', () => {
                 video.play();
-                resolve();
+                resolve(stream);
             });
         });
     })

src/input/config_factory.js

@@ -36,7 +36,7 @@ export function createConfigFromSource(config, sourceConfig, source) {
     }
 }
 
-function createConfigForImage(config, source, inputConfig = {}) {
+export function createConfigForImage(config, source, inputConfig = {}) {
     const staticImageConfig = {
         inputStream: merge({
             type: "ImageStream",