first

Author: w-okada

001_bodypix-worker-js/.gitignore

@@ -0,0 +1,5 @@
+*~
+*#
+node_modules
+dist
+yarn-error.log

001_bodypix-worker-js/.npmignore

@@ -0,0 +1,6 @@
+*~
+*#
+node_modules
+demo
+src
+yarn-error.log

001_bodypix-worker-js/package-lock.json

@@ -0,0 +1,46 @@
+{
+   "name": "@dannadori/bodypix-worker-js",
+   "version": "1.0.18",
+   "description": "",
+   "main": "dist/bodypix-worker.js",
+   "scripts": {
+      "test": "echo \"Error: no test specified\" && exit 1",
+      "webpack": "npx webpack --config webpack.config.js",
+      "clean": "rimraf dist/*",
+      "build": "cp ../common/*.ts ./src;npm-run-all clean webpack"
+   },
+   "repository": {
+      "type": "git",
+      "url": "git+https://github.com/FLECT-DEV-TEAM/image-analyze-workers"
+   },
+   "keywords": [
+      "webworker",
+      "image processing",
+      "tensorflowjs"
+   ],
+   "author": "[email protected]",
+   "license": "MIT",
+   "bugs": {
+      "url": "https://github.com/FLECT-DEV-TEAM/image-analyze-workers/issues"
+   },
+   "homepage": "https://github.com/FLECT-DEV-TEAM/image-analyze-workers#readme",
+   "devDependencies": {
+      "npm-run-all": "^4.1.5",
+      "rimraf": "^3.0.2",
+      "ts-loader": "^8.0.4",
+      "tsconfig-paths": "^3.9.0",
+      "typescript": "^4.0.3",
+      "webpack": "^4.44.2",
+      "webpack-cli": "^3.3.12",
+      "worker-plugin": "^5.0.0"
+   },
+   "dependencies": {
+      "@tensorflow-models/body-pix": "^2.0.5",
+      "@tensorflow/tfjs": "^2.5.0",
+      "@tensorflow/tfjs-backend-wasm": "^2.5.0",
+      "@tensorflow/tfjs-converter": "~1.2.1",
+      "@tensorflow/tfjs-core": "~1.2.1",
+      "@types/node": "^14.11.2",
+      "await-semaphore": "^0.1.3"
+   }
+}

001_bodypix-worker-js/package.json

@@ -0,0 +1,32 @@
+/************************************************
+ * THIS FILE SHOULD BE EDITED IN COMMON FOLDER. *
+ ************************************************/
+
+export enum BrowserType {
+    "MSIE",
+    "EDGE",
+    "CHROME",
+    "SAFARI",
+    "FIREFOX",
+    "OPERA",
+    "OTHER",
+}
+
+export const getBrowserType = () => {
+    var userAgent = window.navigator.userAgent.toLowerCase();
+    if (userAgent.indexOf('msie') !== -1 || userAgent.indexOf('trident') !== -1) {
+        return BrowserType.MSIE
+    } else if (userAgent.indexOf('edge') !== -1) {
+        return BrowserType.EDGE
+    } else if (userAgent.indexOf('chrome') !== -1) {
+        return BrowserType.CHROME
+    } else if (userAgent.indexOf('safari') !== -1) {
+        return BrowserType.SAFARI
+    } else if (userAgent.indexOf('firefox') !== -1) {
+        return BrowserType.FIREFOX
+    } else if (userAgent.indexOf('opera') !== -1) {
+        return BrowserType.OPERA
+    } else {
+        return BrowserType.OTHER
+    }
+}

001_bodypix-worker-js/src/BrowserUtil.ts

@@ -0,0 +1,5 @@
+/************************************************
+ * THIS FILE SHOULD BE EDITED IN COMMON FOLDER. *
+ ************************************************/
+
+export const IMAGE_PATH = "https://www.sponichi.co.jp/entertainment/news/2019/10/04/jpeg/20191004s00041000331000p_view.jpg"

001_bodypix-worker-js/src/DemoUtil.ts

@@ -0,0 +1,183 @@
+import { WorkerResponse, WorkerCommand, BodypixFunctionType, BodyPixConfig, ModelConfigMobileNetV1_05, BodyPixOperatipnParams } from "./const"
+import * as bodyPix from '@tensorflow-models/body-pix'
+import { BrowserType, getBrowserType } from "./BrowserUtil";
+
+export { ModelConfigResNet, ModelConfigMobileNetV1, ModelConfigMobileNetV1_05, BodypixFunctionType } from './const'
+export { BrowserType, getBrowserType} from './BrowserUtil';
+export { IMAGE_PATH } from "./DemoUtil"
+export {SemanticPersonSegmentation, SemanticPartSegmentation, PersonSegmentation, PartSegmentation} from '@tensorflow-models/body-pix'
+export { BodyPixInternalResolution } from '@tensorflow-models/body-pix/dist/types';
+
+export const generateBodyPixDefaultConfig = ():BodyPixConfig => {
+    const defaultConf:BodyPixConfig = {
+        browserType         : getBrowserType(),
+        model               : ModelConfigMobileNetV1_05,
+        processOnLocal      : false,
+    }
+    return defaultConf
+}
+
+export const generateDefaultBodyPixParams = () =>{
+    const defaultParams: BodyPixOperatipnParams = {
+        type: BodypixFunctionType.SegmentPerson,
+        segmentPersonParams:{
+            flipHorizontal : false,
+            internalResolution: "medium",
+            segmentationThreshold: 0.7,
+            maxDetections: 10,
+            scoreThreshold: 0.3,
+            nmsRadius: 20,
+        },
+        segmentPersonPartsParams:{
+            flipHorizontal : false,
+            internalResolution: "medium",
+            segmentationThreshold: 0.7,
+            maxDetections: 10,
+            scoreThreshold: 0.3,
+            nmsRadius: 20,
+        },
+        segmentMultiPersonParams:{
+            flipHorizontal : false,
+            internalResolution: "medium",
+            segmentationThreshold: 0.7,
+            maxDetections: 10,
+            scoreThreshold: 0.3,
+            nmsRadius: 20,
+            minKeypointScore: 0.3,
+            refineSteps:10
+        },
+        segmentMultiPersonPartsParams:{
+            flipHorizontal : false,
+            internalResolution: "medium",
+            segmentationThreshold: 0.7,
+            maxDetections: 10,
+            scoreThreshold: 0.3,
+            nmsRadius: 20,
+            minKeypointScore: 0.3,
+            refineSteps:10
+        },
+        processWidth        : 300,
+        processHeight       : 300,
+
+    }
+    return defaultParams
+}
+
+class LocalBP {
+    model: bodyPix.BodyPix | null = null
+    canvas: HTMLCanvasElement = (()=>{
+        const newCanvas = document.createElement("canvas")
+        newCanvas.style.display="none"
+        return newCanvas
+    })()
+    
+    init = (config: BodyPixConfig) => {
+        return bodyPix.load(config.model).then(res => {
+            console.log("bodypix loaded locally", config)
+            this.model = res
+            return
+        })
+    }
+
+
+    predict = async (targetCanvas: HTMLCanvasElement, config:BodyPixConfig, params:BodyPixOperatipnParams) => {
+        // ImageData作成
+        const processWidth = (params.processWidth <= 0 || params.processHeight <= 0) ? targetCanvas.width : params.processWidth
+        const processHeight = (params.processWidth <= 0 || params.processHeight <= 0) ? targetCanvas.height : params.processHeight
+
+        //console.log("process image size:", processWidth, processHeight)
+        this.canvas.width = processWidth
+        this.canvas.height = processHeight
+        const ctx = this.canvas.getContext("2d")!
+        ctx.drawImage(targetCanvas, 0, 0, processWidth, processHeight)
+        const newImg = ctx.getImageData(0, 0, processWidth, processHeight)
+      
+        let prediction
+        if(params.type === BodypixFunctionType.SegmentPerson){
+          prediction = await this.model!.segmentPerson(newImg, params.segmentPersonParams)
+        }else if(params.type === BodypixFunctionType.SegmentPersonParts){
+          prediction = await this.model!.segmentPersonParts(newImg, params.segmentPersonPartsParams)
+        }else if(params.type === BodypixFunctionType.SegmentMultiPerson){
+          prediction = await this.model!.segmentMultiPerson(newImg, params.segmentMultiPersonParams)
+        }else if(params.type === BodypixFunctionType.SegmentMultiPersonParts){
+          prediction = await this.model!.segmentMultiPersonParts(newImg, params.segmentMultiPersonPartsParams)
+        }else{// segmentPersonに倒す
+          prediction = await this.model!.segmentPerson(newImg, params.segmentPersonParams)
+        }
+        return prediction
+      }
+}
+
+export class BodypixWorkerManager {
+    private workerBP: Worker | null = null
+
+    config:BodyPixConfig = generateBodyPixDefaultConfig()
+    private localBP = new LocalBP()
+
+    init(config:BodyPixConfig|null = null) {
+        if(config != null){
+            this.config = config
+        }
+        if(this.workerBP){
+            this.workerBP.terminate()
+        }
+
+        if(this.config.browserType === BrowserType.SAFARI || this.config.processOnLocal == true){
+            // safariはwebworkerでWebGLが使えないのでworkerは使わない。
+            return new Promise((onResolve, onFail) => {
+                this.localBP.init(this.config!).then(() => {
+                    onResolve()
+                })
+            })
+        }
+        
+
+        this.workerBP = new Worker('./workerBP.ts', { type: 'module' })
+
+        this.workerBP!.postMessage({ message: WorkerCommand.INITIALIZE, config: this.config })
+        return new Promise((onResolve, onFail) => {
+            this.workerBP!.onmessage = (event) => {
+                if (event.data.message === WorkerResponse.INITIALIZED) {
+                    console.log("WORKERSS INITIALIZED")
+                    onResolve()
+                } else {
+                    console.log("Bodypix Initialization something wrong..")
+                    onFail(event)
+                }
+            }
+        })
+    }
+
+    predict(targetCanvas: HTMLCanvasElement, params:BodyPixOperatipnParams) {
+        if(this.config.browserType === BrowserType.SAFARI || this.config.processOnLocal == true){
+            const p = new Promise(async (onResolve: (v: any) => void, onFail) => {
+                const prediction = await this.localBP.predict(targetCanvas, this.config, params)
+                onResolve(prediction)
+            })
+            return p
+        }else{
+            const offscreen = new OffscreenCanvas(targetCanvas.width, targetCanvas.height)
+            const offctx = offscreen.getContext("2d")!
+            offctx.drawImage(targetCanvas, 0, 0, targetCanvas.width, targetCanvas.height)
+            const imageBitmap = offscreen.transferToImageBitmap()
+            const uid = performance.now()
+            const p = new Promise((onResolve: (v:any) => void, onFail) => {
+                this.workerBP!.postMessage({
+                    message: WorkerCommand.PREDICT, uid: uid,
+                    params:params, image: imageBitmap,
+                    config:this.config
+                }, [imageBitmap])
+
+                this.workerBP!.onmessage = (event) => {
+                    if (event.data.message === WorkerResponse.PREDICTED && event.data.uid === uid) {
+                        onResolve(event.data.prediction)
+                    } else {
+                        console.log("Bodypix Prediction something wrong..")
+                        onFail(event)
+                    }
+                }
+            })
+            return p
+        }
+    }
+}

001_bodypix-worker-js/src/bodypix-worker.ts

@@ -0,0 +1,56 @@
+import { ModelConfig, PersonInferenceConfig, MultiPersonInstanceInferenceConfig } from '@tensorflow-models/body-pix/dist/body_pix_model';
+import { BrowserType } from './BrowserUtil';
+
+
+export const WorkerCommand = {
+    INITIALIZE   : 'initialize',
+    PREDICT : 'predict',
+}
+
+export const WorkerResponse = {
+    INITIALIZED      : 'initialized',
+    PREDICTED   : 'predicted',
+}
+
+export interface BodyPixConfig{
+    browserType         : BrowserType
+    model               : ModelConfig
+    processOnLocal      : boolean
+}
+
+export const ModelConfigResNet: ModelConfig = {
+    architecture: 'ResNet50',
+    outputStride: 32,
+    quantBytes: 2
+}
+export const ModelConfigMobileNetV1: ModelConfig = {
+    architecture: 'MobileNetV1',
+    outputStride: 16,
+    multiplier: 0.75,
+    quantBytes: 2
+}
+export const ModelConfigMobileNetV1_05: ModelConfig = {
+    architecture: 'MobileNetV1',
+    outputStride: 16,
+    multiplier: 0.5,
+    quantBytes: 2
+}
+
+export interface BodyPixOperatipnParams{
+    type: BodypixFunctionType
+    segmentPersonParams: PersonInferenceConfig
+    segmentPersonPartsParams: PersonInferenceConfig
+    segmentMultiPersonParams: MultiPersonInstanceInferenceConfig
+    segmentMultiPersonPartsParams: MultiPersonInstanceInferenceConfig
+    processWidth        : number
+    processHeight       : number
+
+}
+
+export enum BodypixFunctionType{
+    SegmentPerson,
+    SegmentMultiPerson,
+    SegmentPersonParts,
+    SegmentMultiPersonParts
+}
+