diff --git a/example/live_w_locator.js b/example/live_w_locator.js index 0589ce0..4e63d13 100644 --- a/example/live_w_locator.js +++ b/example/live_w_locator.js @@ -146,10 +146,11 @@ $(function() { inputStream: { type : "LiveStream", constraints: { - width: {ideal: 640}, + width: {ideal: 480}, height: {ideal: 480}, + zoom: {exact: 2}, facingMode: "environment", - aspectRatio: {min: 1, max: 2} + aspectRatio: 1, } }, locator: { @@ -157,10 +158,10 @@ $(function() { halfSample: true }, numOfWorkers: 2, - frequency: 10, + frequency: 2, decoder: { readers : [{ - format: "code_128_reader", + format: "ean_reader", config: {} }] }, diff --git a/src/common/buffers.js b/src/common/buffers.js new file mode 100644 index 0000000..817230a --- /dev/null +++ b/src/common/buffers.js @@ -0,0 +1,45 @@ +import {DEBUG, log} from './log'; + +const debug = log.bind(null, DEBUG, "buffers.js"); +let buffers = []; + +export function aquire(bytes) { + const allocation = findBySize(bytes); + if (allocation) { + debug(`reusing ${bytes}`, debugSize); + return allocation; + } + debug(`allocating ${bytes}`, debugSize); + const buffer = new ArrayBuffer(bytes); + return buffer; +} + +export function release(buffer) { + if (!buffer) { + throw new Error("Buffer not defined"); + } + buffers.push(buffer); + debug('release', debugSize); +} + +export function releaseAll() { + buffers = []; +} + +function debugSize() { + return "size: " + Object + .keys(buffers) + .filter((key) => buffers[key] !== null) + .length; +} + +function findBySize(bytes) { + for (let i = 0; i < buffers.length; i++) { + if (buffers[i].byteLength === bytes) { + let allocation = buffers[i]; + buffers.splice(i, 1); + return allocation; + } + } + return null; +} diff --git a/src/common/log.js b/src/common/log.js new file mode 100644 index 0000000..79f33d0 --- /dev/null +++ b/src/common/log.js @@ -0,0 +1,16 @@ +export const DEBUG = "debug"; + +export function log(level, scope, ...rest) { + if (level !== DEBUG) { + console.log(`${level}: ${scope} - ${msg(rest)}`); + } +} + +function msg(args) { + return args.map(arg => { + if (typeof arg === 'function') { + return arg(); + } + return arg; + }).join(', '); +} diff --git a/src/config/config.dev.js b/src/config/config.dev.js index f388b7e..5792cfd 100644 --- a/src/config/config.dev.js +++ b/src/config/config.dev.js @@ -2,6 +2,7 @@ module.exports = { numOfWorkers: 0, locate: true, target: '#interactive.viewport', + frequency: 5, constraints: { width: 640, height: 640, diff --git a/src/input/PixelCapture.js b/src/input/PixelCapture.js index efcdf64..e51396e 100644 --- a/src/input/PixelCapture.js +++ b/src/input/PixelCapture.js @@ -1,8 +1,10 @@ +import {memoize} from 'lodash'; import { - computeGray + computeGray, + computeImageArea, } from '../common/cv_utils'; -import {sleep} from '../common/utils'; -import {getViewport} from '../common/utils'; +import {sleep, getViewport} from '../common/utils'; +import {aquire} from '../common/buffers'; function adjustCanvasSize(input, canvas) { if (input instanceof HTMLVideoElement) { @@ -43,7 +45,6 @@ export function fromSource(source, {target = "#interactive.viewport"} = {}) { var drawable = source.getDrawable(); var $canvas = null; var ctx = null; - var bytePool = []; if (drawable instanceof HTMLVideoElement || drawable instanceof HTMLImageElement) { @@ -56,24 +57,14 @@ export function fromSource(source, {target = "#interactive.viewport"} = {}) { ctx = drawable.getContext('2d'); } - function nextAvailableBuffer() { - var i; - var buffer; - var bytesRequired = ($canvas.height * $canvas.width); - for (i = 0; i < bytePool.length; i++) { - buffer = bytePool[i]; - if (buffer && buffer.buffer.byteLength === bytesRequired) { - return bytePool[i]; - } - } - buffer = new Uint8Array(bytesRequired); - bytePool.push(buffer); - console.log("Added new entry to bufferPool", bytesRequired); - return buffer; + function nextAvailableBuffer(bytesRequired) { + return new Uint8Array(aquire(bytesRequired)); } + + return { - grabFrameData: function grabFrameData({buffer, clipping}) { + grabFrameData: function grabFrameData({clipping} = {}) { const {viewport, canvas: canvasSize} = source.getDimensions(); const sx = viewport.x; const sy = viewport.y; @@ -84,20 +75,38 @@ export function fromSource(source, {target = "#interactive.viewport"} = {}) { const dWidth = canvasSize.width; const dHeight = canvasSize.height; + console.time("clipp") + + clipping = clipping ? clipping(canvasSize) : { + x: 0, + y: 0, + width: canvasSize.width, + height: canvasSize.height, + }; + adjustCanvasSize(canvasSize, $canvas); if ($canvas.height < 10 || $canvas.width < 10) { - console.log('$canvas not initialized. Waiting 100ms and then continuing'); return sleep(100).then(grabFrameData); } if (!(drawable instanceof HTMLCanvasElement)) { ctx.drawImage(drawable, sx, sy, sWidth, sHeight, dx, dy, dWidth, dHeight); } - var imageData = ctx.getImageData(0, 0, $canvas.width, $canvas.height).data; - var imageBuffer = buffer ? buffer : nextAvailableBuffer(); + var imageData = ctx.getImageData( + clipping.x, + clipping.y, + clipping.width, + clipping.height + ).data; + var imageBuffer = nextAvailableBuffer(clipping.width * clipping.height); computeGray(imageData, imageBuffer); return Promise.resolve({ - width: $canvas.width, - height: $canvas.height, + width: clipping.width, + height: clipping.height, + dimensions: { + viewport, + canvas: canvasSize, + clipping, + }, data: imageBuffer, }); }, diff --git a/src/locator/barcode_locator.js b/src/locator/barcode_locator.js index 58e6968..4bb949c 100644 --- a/src/locator/barcode_locator.js +++ b/src/locator/barcode_locator.js @@ -14,6 +14,11 @@ import ImageDebug from '../common/image_debug'; import Rasterizer from './rasterizer'; import Tracer from './tracer'; import skeletonizer from './skeletonizer'; +import {DEBUG, log} from '../common/log'; + +const debug = log.bind(null, DEBUG, "barcode_locator.js"); + + const vec2 = { clone: require('gl-vec2/clone'), dot: require('gl-vec2/dot'), @@ -563,38 +568,44 @@ export default function createLocator(inputImageWrapper, config) { } }; } -export function checkImageConstraints(inputStream, config) { +export function checkImageConstraints({canvasSize, area, patchSize, halfSample: half}) { var patchSize, - width = inputStream.getWidth(), - height = inputStream.getHeight(), - halfSample = config.halfSample ? 0.5 : 1, + width = canvasSize.width, + height = canvasSize.height, + half = half ? 0.5 : 1, size, - area; - - // calculate width and height based on area - if (inputStream.getConfig().area) { - area = computeImageArea(width, height, inputStream.getConfig().area); - inputStream.setTopRight({x: area.sx, y: area.sy}); - inputStream.setCanvasSize({x: width, y: height}); - width = area.sw; - height = area.sh; + clipping = { + x: 0, + y: 0, + width, + height, + }; + + if (area) { + const imageArea = computeImageArea(width, height, area); + clipping.x = imageArea.sx; + clipping.y = imageArea.sy; + clipping.width = width = imageArea.sw; + clipping.height = height = imageArea.sh; } size = { - x: Math.floor(width * halfSample), - y: Math.floor(height * halfSample) + x: Math.floor(width * half), + y: Math.floor(height * half) }; - patchSize = calculatePatchSize(config.patchSize, size); + patchSize = calculatePatchSize(patchSize, size); if (ENV.development) { - console.log("Patch-Size: " + JSON.stringify(patchSize)); + debug("Patch-Size: " + JSON.stringify(patchSize)); } - inputStream.setWidth(Math.floor(Math.floor(size.x / patchSize.x) * (1 / halfSample) * patchSize.x)); - inputStream.setHeight(Math.floor(Math.floor(size.y / patchSize.y) * (1 / halfSample) * patchSize.y)); + clipping.width = Math.floor(Math.floor(size.x / patchSize.x) * (1 / half) * patchSize.x); + clipping.height = Math.floor(Math.floor(size.y / patchSize.y) * (1 / half) * patchSize.y); + clipping.x = Math.floor((canvasSize.width - clipping.width) / 2); + clipping.y = Math.floor((canvasSize.height - clipping.height) / 2); - if ((inputStream.getWidth() % patchSize.x) === 0 && (inputStream.getHeight() % patchSize.y) === 0) { - return true; + if ((clipping.width % patchSize.x) === 0 && (clipping.height % patchSize.y) === 0) { + return clipping; } throw new Error("Image dimensions do not comply with the current settings: Width (" + diff --git a/src/scanner.js b/src/scanner.js index 062b2e0..adc1571 100644 --- a/src/scanner.js +++ b/src/scanner.js @@ -1,23 +1,33 @@ +import {merge, memoize} from 'lodash'; + import ImageWrapper from './common/image_wrapper'; import createLocator, {checkImageConstraints} from './locator/barcode_locator'; import BarcodeDecoder from './decoder/barcode_decoder'; import createEventedElement from './common/events'; -import CameraAccess from './input/camera_access'; -import ImageDebug from './common/image_debug'; -import ResultCollector from './analytics/result_collector'; +import {release, aquire, releaseAll} from './common/buffers'; import Config from './config/config'; -import InputStream from 'input_stream'; -import FrameGrabber from 'frame_grabber'; -import {merge} from 'lodash'; +import CameraAccess from './input/camera_access'; + + + const vec2 = { clone: require('gl-vec2/clone') }; +const getDecoder = memoize((decoderConfig, _inputImageWrapper) => { + return BarcodeDecoder.create(decoderConfig, _inputImageWrapper); +}, (decoderConfig, _inputImageWrapper) => { + return JSON.stringify(Object.assign({}, decoderConfig, {width: _inputImageWrapper.size.x, height: _inputImageWrapper.size.y})); +}); + +const _checkImageConstraints = memoize((opts) => { + return checkImageConstraints(opts); +}, (opts) => { + return JSON.stringify(opts); +}); function createScanner(pixelCapturer) { - var _inputStream, - _framegrabber, - _stopped = true, + var _stopped = true, _canvasContainer = { ctx: { image: null @@ -28,7 +38,6 @@ function createScanner(pixelCapturer) { }, _inputImageWrapper, _boxSize, - _decoder, _workerPool = [], _onUIThread = true, _resultCollector, @@ -39,20 +48,14 @@ function createScanner(pixelCapturer) { const source = pixelCapturer ? pixelCapturer.getSource() : {}; function setup() { - // checkImageConstraints(_inputStream, _config.locator); return adjustWorkerPool(_config.numOfWorkers) .then(() => { if (_config.numOfWorkers === 0) { - initializeData(); + initBuffers(); } }); } - function initializeData(imageWrapper) { - initBuffers(imageWrapper); - _decoder = BarcodeDecoder.create(_config.decoder, _inputImageWrapper); - } - function initBuffers(imageWrapper) { if (imageWrapper) { _inputImageWrapper = imageWrapper; @@ -174,7 +177,8 @@ function createScanner(pixelCapturer) { boxes = getBoundingBoxes(); if (boxes) { - result = _decoder.decodeFromBoundingBoxes(boxes); + result = getDecoder(_config.decoder, _inputImageWrapper) + .decodeFromBoundingBoxes(boxes); result = result || {}; result.boxes = boxes; publishResult(result, _inputImageWrapper.data); @@ -183,6 +187,14 @@ function createScanner(pixelCapturer) { } } + function calculateClipping(canvasSize) { + const area = _config.detector.area; + const patchSize = _config.locator.patchSize || "medium"; + const halfSample = _config.locator.halfSample || true; + + return _checkImageConstraints({area, patchSize, canvasSize, halfSample}); + } + function update() { var availableWorker; @@ -195,17 +207,20 @@ function createScanner(pixelCapturer) { return Promise.resolve(); } } - const buffer = availableWorker ? availableWorker.imageData : _inputImageWrapper.data; - return pixelCapturer.grabFrameData({buffer}) + return pixelCapturer.grabFrameData({clipping: calculateClipping}) .then((bitmap) => { if (bitmap) { + console.log(bitmap.dimensions); + // adjust image size! if (availableWorker) { + availableWorker.imageData = bitmap.data; availableWorker.busy = true; availableWorker.worker.postMessage({ cmd: 'process', imageData: availableWorker.imageData }, [availableWorker.imageData.buffer]); } else { + _inputImageWrapper.data = bitmap.data; locateAndDecode(); } } @@ -250,7 +265,7 @@ function createScanner(pixelCapturer) { const captureSize = pixelCapturer.getCaptureSize(); const workerThread = { worker: undefined, - imageData: new Uint8Array(captureSize.width * captureSize.height), + imageData: new Uint8Array(aquire(captureSize.width * captureSize.height)), busy: true }; @@ -261,13 +276,13 @@ function createScanner(pixelCapturer) { if (e.data.event === 'initialized') { URL.revokeObjectURL(blobURL); workerThread.busy = false; - workerThread.imageData = new Uint8Array(e.data.imageData); + release(e.data.imageData); if (ENV.development) { console.log("Worker initialized"); } return cb(workerThread); } else if (e.data.event === 'processed') { - workerThread.imageData = new Uint8Array(e.data.imageData); + release(e.data.imageData); workerThread.busy = false; publishResult(e.data.result, workerThread.imageData); } else if (e.data.event === 'error') { @@ -350,16 +365,6 @@ function createScanner(pixelCapturer) { return window.URL.createObjectURL(blob); } - function setReaders(readers) { - if (_decoder) { - _decoder.setReaders(readers); - } else if (_onUIThread && _workerPool.length > 0) { - _workerPool.forEach(function(workerThread) { - workerThread.worker.postMessage({cmd: 'setReaders', readers: readers}); - }); - } - } - function adjustWorkerPool(capacity) { return new Promise((resolve) => { const increaseBy = capacity - _workerPool.length; @@ -397,7 +402,7 @@ function createScanner(pixelCapturer) { if (imageWrapper) { _onUIThread = false; - initializeData(imageWrapper); + initBuffers(imageWrapper); return cb(); } else { return setup().then(cb); @@ -412,6 +417,7 @@ function createScanner(pixelCapturer) { stop: function() { _stopped = true; adjustWorkerPool(0); + releaseAll(); if (source.type === "CAMERA") { CameraAccess.release(); }