Merge pull request #82 from habbes/dev

v0.7

README.md

@@ -2,25 +2,25 @@
 
 Interactive web-based playground for computer vision and image processing, useful for experiments and exploration.
 
-[https://xaval.habbes.xyz](https://xaval.habbes.xyz)
+[https://xaval.app](https://xaval.app)
 
 ## Current Features
 
 - In-browser JS environment where you write your code
 - [OpenCV](https://docs.opencv.org/3.4.1/d5/d10/tutorial_js_root.html) is preloaded
 - You can import multiple images from your computer
-- Simple API for loading input images and displaying out images
+- Image viewer for displaying output with simple API
+- Importing images and videos
+- Importing and working with arbitrary file blobs
 - Custom widgets supporting multiple control types and pipelines that control function parameters in real time
-- Easy API for working with video streams from a connected camera
+- Easy API for working with video streams from a connected camera or imported video
 
 ## Planned Features
 
 In no particular order, here are some of the features planned:
 
 - Support for Tensorflow.js
-- Support for different file types besides images
 - Load images/videos from the web
-- Support for importing videos
 - Background code execution
 - Console/Stdout emulator
 - Multiple code cells

src/core/files/types.ts

@@ -1,22 +1,94 @@
+import { VideoSource } from '@/core/video';
+
+/**
+ * represents a library for managing files,
+ * it provides access to add and read files of
+ * different types
+ * each file has unique name
+ * the implementation is also assumed to track
+ * metadata about each type internally (e.g. type)
+ */
 export interface FileLibrary {
+
     /**
      * adds an image to the file source
      * @param fileUrl object url of the image
      * @param filename optional name for the file, if not provided, one will be auto-assigned
      */
     addImage(fileUrl: string, filename?: string): any;
 
+    /**
+     * adds a video to the file library
+     * @param fileUrl object url of the image
+     * @param filename optional name for the file, if not provided, one will be auto-assigned
+     */
+    addVideo(fileUrl: string, filename?: string): any;
+
+    /**
+     * adds a file of an arbitrary type
+     * @param file blob containing the file
+     * @param filename optional name assigned to the file in the library
+     */
+    addBinary(file: Blob, filename?: string): any;
+
     /**
      * reads a specified image from the library as an OpenCv matrix
      * @param name file name
      * @returns {cv.Mat} OpenCV matrix containing the image
      */
     readImage(name: string): any;
 
+    /**
+     * reads specified video from the library
+     * @param name file name
+     * @returns video
+     */
+    readVideo(name: string): VideoSource;
+
+    /**
+     * returns a reader to access the specified file as a blob
+     * @param name 
+     */
+    getReader(name: string): BinaryFileReader;
+
+    /**
+     * reads file based on its type
+     * @param name file name
+     * @returns depends on file type
+     */
+    read(name: string): any;
+
     /**
      * changes the name of a file in the library
      * @param oldName current name of the file
      * @param newName new name, should be unique in the library
      */
     rename(oldName: string, newName: string): any;
-}
+}
+
+export type FileType = 'image' | 'video' | 'binary';
+
+/**
+ * represents a wrapper around a blob
+ */
+export interface BinaryFileReader {
+    /**
+     * the object url of the file.
+     */
+    url: string;
+
+    /**
+     * reads the contents of the file as text
+     */
+    readText(): Promise<any>;
+
+    /**
+     * returns the contents of the file as an ArrayBuffer
+     */
+    readBuffer(): Promise<any>;
+
+    /**
+     * returns the contents of the file as data url
+     */
+    readDataURL(): Promise<any>;
+}

src/core/util/index.ts

@@ -0,0 +1 @@
+export { applyMixins } from './mixins';

src/core/util/mixins.ts

@@ -0,0 +1,12 @@
+/**
+ * applies mixins to a given class
+ * @param derivedCtor target class
+ * @param baseCtors mixins
+ */
+export function applyMixins(derivedCtor: any, baseCtors: any[]) {
+    baseCtors.forEach(baseCtor => {
+        Object.getOwnPropertyNames(baseCtor.prototype).forEach(name => {
+            derivedCtor.prototype[name] = baseCtor.prototype[name];
+        });
+    });
+}

src/core/widget/types.ts

@@ -90,6 +90,7 @@ export interface WidgetModel extends DataSource<WidgetUpdateResult> {
      * @internal
      */
     _outputsObservables: {[name: string]: Observable<any>};
+    _paramUpdateHandler?: WidgetParamUpdateHandler;
     /**
      * updates the specified input
      * this updates the widget
@@ -126,6 +127,12 @@ export interface WidgetModel extends DataSource<WidgetUpdateResult> {
      * @param paramName
      */
     getParam(paramName: string): any;
+    /**
+     * registers a handler to be called when a param
+     * has been updated
+     * @param handler
+     */
+    onParamUpdated (handler: WidgetParamUpdateHandler): any;
     /**
      * updates the widget
      */
@@ -177,3 +184,7 @@ export interface WidgetTemplate {
     opts: WidgetOpts;
     create(): WidgetModel;
 }
+
+export interface WidgetParamUpdateHandler {
+    (paramName: string, value: any): any;
+}

src/core/widget/widget-model.ts

@@ -24,6 +24,7 @@ export function createWidgetCreateFunction (opts: WidgetOpts)  {
             inputs: {},
             outputs: {},
             _outputsObservables: {},
+            _paramUpdateHandler: undefined,
             getInput (name: string) {
                 return this.state.inputs[name];
             },
@@ -42,18 +43,30 @@ export function createWidgetCreateFunction (opts: WidgetOpts)  {
             },
             setParam (name: string, value: any) {
                 this.state.params[name] = value;
+                if (this._paramUpdateHandler) {
+                    this._paramUpdateHandler(name, value);
+                }
                 this.update();
             },
             setParams (params) {
                 Object.keys(params).forEach(name => {
                     this.state.params[name] = params[name];
+                    if (this._paramUpdateHandler) {
+                        this._paramUpdateHandler(name, params[name]);
+                    }
                 });
                 this.update();
             },
+            onParamUpdated (handler) {
+                this._paramUpdateHandler = handler;
+            },
             get observable () {
                 return source;
             },
             update () {
+                if (!validateUpdate(this)) {
+                    return;
+                }
                 const outputs = this.opts.onUpdate(this.state);
                 source.next(outputs);
             },
@@ -111,6 +124,21 @@ function setupOutputDataSources (widget: WidgetModel) {
     });
 }
 
+/**
+ * checks whether the widget state is in a valid
+ * state for the update callback to run
+ * @param widget 
+ */
+function validateUpdate (widget: WidgetModel): boolean {
+    const res = !(
+        Object.keys(widget.opts.inputs)
+        .some(input => typeof widget.state.inputs[input] === 'undefined')
+        || Object.keys(widget.opts.params)
+        .some(param => typeof widget.state.params[param] === 'undefined')
+    );
+    return res;
+}
+
 /**
  * initializes a widget model's state
  * @param opts 
@@ -126,7 +154,7 @@ export function initWidgetModelState (opts: WidgetOpts): WidgetModelContext {
         state.params[paramName] = value;
     }
     for (let inputName in opts.inputs) {
-        state.inputs[inputName] = null;
+        state.inputs[inputName] = undefined;
     }
     return state;
 }
@@ -145,7 +173,7 @@ export function getDefaultInitialValueForType (type: WidgetArgDataType): any {
         case WidgetArgDataType.String:
             return '';
         default:
-            return null;
+            return undefined;
     }
 }
 

src/index.ts

@@ -4,3 +4,9 @@ window.onload = () => {
     const app = init();
     app.start();
 };
+
+// prevent accidentally closing the app, warn user before exit if changes were made
+window.addEventListener('beforeunload', (e) => {
+    (e || window.event).returnValue = true;
+    return true;
+});

src/init.ts

@@ -34,6 +34,7 @@ export default function init () {
     samples.addSamples({
         'Quick Intro': codeSamples.QUICK_INTRO,
         'Widgets': codeSamples.WIDGETS,
+        'Video': codeSamples.VIDEO,
         'Camera': codeSamples.CAMERA,
         'Edge Detection': codeSamples.EDGE_DETECTION
     });

src/samples/index.ts

@@ -1,11 +1,15 @@
 import CAMERA from './camera';
 import EDGE_DETECTION from './edge-detection';
 import QUICK_INTRO from './quick-intro';
+import SIMPLE_GREEN_SCREEN from './simple-green-screen';
+import VIDEO from './video';
 import WIDGETS from './widgets';
 
 export {
     CAMERA,
     EDGE_DETECTION,
     QUICK_INTRO,
+    SIMPLE_GREEN_SCREEN,
+    VIDEO,
     WIDGETS
 };

src/samples/simple-green-screen.ts

@@ -0,0 +1,58 @@
+export default 
+`const { widgets, io: { imageViewer, files, cameras } } = xaval;
+
+const green = files.readVideo('green');
+const greenStream = green.getStream({ fps: 30 });
+const camera = cameras.getDefault();
+const cameraStream = camera.getStream({ fps: 30 });
+
+widgets.define('Blend', {
+  params: {
+    minG: {
+      type: 'number',
+      min: 0,
+      max: 255,
+      initial: 200
+    },
+    maxR: {
+      type: 'number',
+      min: 0,
+      max: 255,
+      initial: 50
+    },
+    maxB: {
+      type: 'number',
+      min: 0,
+      max: 255,
+      initial: 50
+    }
+  },
+  inputs: ['green', 'target'],
+  outputs: ['res'],
+  onUpdate(ctx) {
+    const { green, target } = ctx.inputs;
+    const { minG, maxR, maxB } = ctx.params;
+    const res = new cv.Mat(target.rows, target.cols, target.type());
+    cv.resize(green, res, res.size());
+    for (let row=0; row < res.rows; row++) {
+      for (let col=0; col < res.cols; col++) {
+        const [r, g, b] = res.ucharPtr(row, col);
+        const tp = target.ucharPtr(row, col);
+        if (r < maxR && b < maxB && g > minG) {
+          res.data.set(tp, row * res.cols * res.channels() + col * res.channels());
+        }
+      }
+    }
+    return { res };
+  }
+});
+
+const widget = widgets.create('Blend');
+widget.outputs.res.pipe(imageViewer);
+greenStream.pipe(widget.inputs.green);
+cameraStream.pipe(widget.inputs.target);
+camera.start();
+green.looping = true;
+green.play();
+`;
+

src/samples/video.ts

@@ -0,0 +1,18 @@
+export default
+`const { files, imageViewer } = xaval.io;
+
+// import video file in the file library
+
+// read imported video
+const video = files.readVideo('file1');
+
+// get video stream and attach it to the image viewer
+const stream = video.getStream({ fps: 30 });
+stream.pipe(imageViewer);
+
+// enable the following line if you want the video to loop
+// video.looping = true
+
+// play the video
+video.play();
+`;