diff --git a/extensions/AugmentedReality/index.js b/extensions/AugmentedReality/index.js index 7a444e8..ec2789b 100644 --- a/extensions/AugmentedReality/index.js +++ b/extensions/AugmentedReality/index.js @@ -1,33 +1,38 @@ (async function () { - let videoMirrored = false; - const dictionaries = ['APRILTAG_16h5', 'APRILTAG_16h5_mini','APRILTAG_16h5_duo']; - - const localhost = window.location.search.includes('localhost'); - const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/'; - - const rendererURL = root + 'extensions/AugmentedReality/js/renderModule.mjs'; - const tagURL = root + 'extensions/AugmentedReality/js/tagHandler.mjs'; + const dictionaries = [ + "APRILTAG_16h5", + "APRILTAG_16h5_mini", + "APRILTAG_16h5_duo", + ]; + + const localhost = window.location.search.includes("localhost"); + const root = localhost + ? "http://localhost:8000/" + : "https://extensions.netsblox.org/"; + + const rendererURL = root + "extensions/AugmentedReality/js/renderModule.mjs"; + const tagURL = root + "extensions/AugmentedReality/js/tagHandler.mjs"; const renderModule = await import(rendererURL); const tagModule = await import(tagURL); - + function snapify(value) { if (Array.isArray(value)) { const res = []; for (const item of value) res.push(snapify(item)); return new List(res); - } else if (typeof(value) === 'object') { + } else if (typeof value === "object") { const res = []; for (const key in value) res.push(new List([key, snapify(value[key])])); return new List(res); } else return value; } - + class AugmentedReality extends Extension { constructor(ide) { - super('AugmentedReality'); + super("AugmentedReality"); this.ide = ide; } @@ -35,168 +40,221 @@ videoMirrored = this.ide.stage.mirrorVideo; } - getMenu() { return { - - 'Code Generator': function () { - new ArucoGenMorph().popUp(world); - }, - - }; } + getMenu() { + return { + "Code Generator": function () { + new ArucoGenMorph().popUp(world); + }, + }; + } - getCategories() { return []; } + getCategories() { + return []; + } getPalette() { const blocks = [ - new Extension.Palette.Block('ARCodeTracker'), - new Extension.Palette.Block('ARCodeRender'), - '-', - new Extension.Palette.Block('ARCodeFlag'), - new Extension.Palette.Block('ARCodeVisibleArray'), - '-', - new Extension.Palette.Block('ARCodeSetDictionary'), - new Extension.Palette.Block('ARCodeDictionary').withWatcherToggle(), - '-', - new Extension.Palette.Block('ARCodeFlipVideo'), - new Extension.Palette.Block('ARCodeFlipped').withWatcherToggle(), - '-' + new Extension.Palette.Block("ARCodeTracker"), + new Extension.Palette.Block("ARCodeRender"), + "-", + new Extension.Palette.Block("ARCodeFlag"), + new Extension.Palette.Block("ARCodeVisibleArray"), + "-", + new Extension.Palette.Block("ARCodeSetDictionary"), + new Extension.Palette.Block("ARCodeDictionary").withWatcherToggle(), + "-", + new Extension.Palette.Block("ARCodeFlipVideo"), + new Extension.Palette.Block("ARCodeFlipped").withWatcherToggle(), + "-", ]; return [ - new Extension.PaletteCategory('sensing', blocks, SpriteMorph), - new Extension.PaletteCategory('sensing', blocks, StageMorph), + new Extension.PaletteCategory("sensing", blocks, SpriteMorph), + new Extension.PaletteCategory("sensing", blocks, StageMorph), ]; } getBlocks() { function block(name, type, category, spec, defaults, action) { - return new Extension.Block(name, type, category, spec, defaults, action).for(SpriteMorph, StageMorph) + return new Extension.Block(name, type, category, spec, defaults, action) + .for(SpriteMorph, StageMorph); } return [ - block('ARCodeTracker', 'reporter', 'sensing', 'find AR code %s', [], function (image) { - return this.runAsyncFn(async () => { - - image = image?.contents || image; - if (!image || typeof(image) !== 'object' || !image.width || !image.height) - throw TypeError('Expected an image as input'); - - const coordinates = await tagModule.getCoordinates(image); - const res = await tagModule.transformCoordinates(coordinates, image); + block( + "ARCodeTracker", + "reporter", + "sensing", + "find AR code %s", + [], + function (image) { + return this.runAsyncFn(async () => { + image = image?.contents || image; + if ( + !image || typeof image !== "object" || !image.width || + !image.height + ) { + throw TypeError("Expected an image as input"); + } - return snapify(res); + const coordinates = await tagModule.getCoordinates(image); + const res = await tagModule.transformCoordinates( + coordinates, + image, + ); - }, { args: [], timeout: 10000 }); - }), - - - block('ARCodeRender', 'reporter', 'sensing', 'render %model on %s', ['box'], function (model, image) { + return snapify(res); + }, { args: [], timeout: 10000 }); + }, + ), + + block("ARCodeRender", "reporter", "sensing", "render %model on %s", [ + "box", + ], function (model, image) { return this.runAsyncFn(async () => { - image = image?.contents || image; - if (!image || typeof(image) !== 'object' || !image.width || !image.height){ - throw TypeError('Expected an image as input'); + if ( + !image || typeof image !== "object" || !image.width || + !image.height + ) { + throw TypeError("Expected an image as input"); } const res = await renderModule.renderScene(image, model); - return new Costume(res); - + return new Costume(res); }, { args: [], timeout: 10000 }); }), - - block('ARCodeFlag', 'predicate', 'sensing', 'AR code %n visible in %s ?', [], function (value, image) { - return this.runAsyncFn(async () => { - - image = image?.contents || image; - if (!image || typeof(image) !== 'object' || !image.width || !image.height){ - throw TypeError('Expected an image as input'); - } - - console.log(value); - value = value?.contents || value; - - if(typeof(value) === 'number'){ - const temp = Array(); - temp.push(value); - value = temp; - } - if (!value || !value.length){ - throw TypeError('Expected number or list'); - } - - for(let i = 0; i < value.length; i++){ - if(typeof(value[i]) === 'string' && !(value[i] = parseInt(value[i]))){ - throw TypeError('list elements must be numbers'); + block( + "ARCodeFlag", + "predicate", + "sensing", + "AR code %n visible in %s ?", + [], + function (value, image) { + return this.runAsyncFn(async () => { + image = image?.contents || image; + if ( + !image || typeof image !== "object" || !image.width || + !image.height + ) { + throw TypeError("Expected an image as input"); } - } - const visible = await tagModule.isTagVisible(image, value); + console.log(value); + value = value?.contents || value; - return snapify(visible); - - }, { args: [], timeout: 10000 }); - }), - - - block('ARCodeVisibleArray', 'reporter', 'sensing', 'All AR codes visible in %s', [], function (image) { - return this.runAsyncFn(async () => { - - image = image?.contents || image; - if (!image || typeof(image) !== 'object' || !image.width || !image.height){ - throw TypeError('Expected an image as input'); - } - - const visible = await tagModule.getVisibleTags(image); - - return snapify(visible); - - }, { args: [], timeout: 10000 }); - }), - - - block('ARCodeSetDictionary', 'command', 'sensing', 'set dictionary to %dictionaries', ['APRILTAG_16h5'], function (dict) { - return this.runAsyncFn(async () => { - - console.log(dict, dictionaries.indexOf(dict) ); - if(dictionaries.indexOf(dict) === -1){ - return new Error(dict, 'is not a valid dictionary.'); - } + if (typeof value === "number") { + const temp = Array(); + temp.push(value); + value = temp; + } + if (!value || !value.length) { + throw TypeError("Expected number or list"); + } - tagModule.setDictionary(dict) + for (let i = 0; i < value.length; i++) { + if ( + typeof (value[i]) === "string" && + !(value[i] = parseInt(value[i])) + ) { + throw TypeError("list elements must be numbers"); + } + } - }, { args: [], timeout: 10000 }); - }), + const visible = await tagModule.isTagVisible(image, value); + + return snapify(visible); + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "ARCodeVisibleArray", + "reporter", + "sensing", + "All AR codes visible in %s", + [], + function (image) { + return this.runAsyncFn(async () => { + image = image?.contents || image; + if ( + !image || typeof image !== "object" || !image.width || + !image.height + ) { + throw TypeError("Expected an image as input"); + } - - block('ARCodeDictionary', 'reporter', 'sensing', 'dictionary', [], function () { - return tagModule.getDictionary(); - }), + const visible = await tagModule.getVisibleTags(image); + + return snapify(visible); + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "ARCodeSetDictionary", + "command", + "sensing", + "set dictionary to %dictionaries", + ["APRILTAG_16h5"], + function (dict) { + return this.runAsyncFn(async () => { + console.log(dict, dictionaries.indexOf(dict)); + if (dictionaries.indexOf(dict) === -1) { + return new Error(dict, "is not a valid dictionary."); + } - - block('ARCodeFlipVideo', 'command', 'sensing', 'flip video', [], function () { - return this.runAsyncFn(async () => { - - world.children[0].stage.mirrorVideo = !world.children[0].stage.mirrorVideo; + tagModule.setDictionary(dict); + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "ARCodeDictionary", + "reporter", + "sensing", + "dictionary", + [], + function () { + return tagModule.getDictionary(); + }, + ), + + block( + "ARCodeFlipVideo", + "command", + "sensing", + "flip video", + [], + function () { + return this.runAsyncFn(async () => { + world.children[0].stage.mirrorVideo = !world.children[0].stage + .mirrorVideo; + videoMirrored = world.children[0].stage.mirrorVideo; + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "ARCodeFlipped", + "reporter", + "sensing", + "flipped?", + [], + function () { videoMirrored = world.children[0].stage.mirrorVideo; - - }, { args: [], timeout: 10000 }); - }), - - - block('ARCodeFlipped', 'reporter', 'sensing', 'flipped?', [], function () { - - videoMirrored = world.children[0].stage.mirrorVideo; - return snapify(videoMirrored); - }) + return snapify(videoMirrored); + }, + ), ]; } - getLabelParts() { function identityMap(s) { const res = {}; for (const x of s) res[x] = x; - return res; + return res; } - + function unionMaps(maps) { const res = {}; for (const map of maps) { @@ -205,33 +263,35 @@ return res; } return [ - new Extension.LabelPart('model', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(['box', 'drum', 'piano']), - true - )), - new Extension.LabelPart('dictionaries', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(dictionaries), - true - )), + new Extension.LabelPart("model", () => + new InputSlotMorph( + null, // text + false, // numeric + identityMap(["box", "drum", "piano"]), + true, + )), + new Extension.LabelPart("dictionaries", () => + new InputSlotMorph( + null, // text + false, // numeric + identityMap(dictionaries), + true, + )), ]; } } - const sources = [root + 'extensions/AugmentedReality/js/ui-morphs.js']; + const sources = [root + "extensions/AugmentedReality/js/ui-morphs.js"]; - for(const source of sources){ - const script = document.createElement('script'); - script.class = 'ARUIScripts'; - script.type = 'text/javascript'; + for (const source of sources) { + const script = document.createElement("script"); + script.class = "ARUIScripts"; + script.type = "text/javascript"; script.src = source; script.async = false; - script.crossOrigin = 'anonymous'; + script.crossOrigin = "anonymous"; document.body.appendChild(script); - } + } NetsBloxExtensions.register(AugmentedReality); disableRetinaSupport(); diff --git a/extensions/AugmentedReality/js/filters.mjs b/extensions/AugmentedReality/js/filters.mjs index 3ce85cd..228969f 100644 --- a/extensions/AugmentedReality/js/filters.mjs +++ b/extensions/AugmentedReality/js/filters.mjs @@ -1,29 +1,29 @@ -import {OneEuroFilter} from 'https://esm.run/@david18284/one-euro-filter@1.0.3'; +import { OneEuroFilter } from "https://esm.run/@david18284/one-euro-filter@1.0.3"; class singleExpFilter { constructor(startVals, alpha) { this.oldVals = startVals; this.a = alpha; } - + filter(...args) { const res = new Array(); const diff = this.oldVals.length - args.length; - if(diff < 0){ - for(let i = args.length; i > this.oldVals.length; i--){ - this.oldVals.push(args[i-1]); - console.log(this.oldVals) + if (diff < 0) { + for (let i = args.length; i > this.oldVals.length; i--) { + this.oldVals.push(args[i - 1]); + console.log(this.oldVals); } } - for(let i = 0; i < args.length; i++){ + for (let i = 0; i < args.length; i++) { const newVal = (this.a * this.oldVals[i]) + ((1 - this.a) * args[i]); res.push(newVal); } this.oldVals = res; - + return res; } } -export {singleExpFilter, OneEuroFilter} \ No newline at end of file +export { OneEuroFilter, singleExpFilter }; diff --git a/extensions/AugmentedReality/js/renderModule.mjs b/extensions/AugmentedReality/js/renderModule.mjs index eea3fda..310354e 100644 --- a/extensions/AugmentedReality/js/renderModule.mjs +++ b/extensions/AugmentedReality/js/renderModule.mjs @@ -1,16 +1,21 @@ -import * as three from 'https://esm.run/three@0.160.0'; -import * as GLTFModule from 'https://esm.run/three@0.160.0/examples/jsm/loaders/GLTFLoader.js'; -import * as handModule from '../../HandGestures/handLandmarkerModule.mjs'; -import * as tagModule from './tagHandler.mjs'; -import * as filters from './filters.mjs'; +import * as three from "https://esm.run/three@0.160.0"; +import * as GLTFModule from "https://esm.run/three@0.160.0/examples/jsm/loaders/GLTFLoader.js"; +import * as handModule from "../../HandGestures/handLandmarkerModule.mjs"; +import * as tagModule from "./tagHandler.mjs"; +import * as filters from "./filters.mjs"; -const localhost = window.location.search.includes('localhost'); -const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/'; +const localhost = window.location.search.includes("localhost"); +const root = localhost + ? "http://localhost:8000/" + : "https://extensions.netsblox.org/"; const models = {}; -function setModelDic(){ - Object.defineProperty(models, 'box', {value: {mesh: createBoxObj(0xaa0000), scalar: 6}, writable: false}); +function setModelDic() { + Object.defineProperty(models, "box", { + value: { mesh: createBoxObj(0xaa0000), scalar: 6 }, + writable: false, + }); console.log(models); createDrumObj(); createPianoObj(); @@ -18,34 +23,51 @@ function setModelDic(){ setModelDic(); -function createDrumObj(){ +function createDrumObj() { const loader = new GLTFModule.GLTFLoader(); - const modelURL = 'https://samankittani.github.io/GLTF_Model_Server/models/drum/untitled.gltf' - - loader.load(modelURL, function ( gltf ) { - const drum = gltf.scene.children[0]; - Object.defineProperty(models, 'drum', {value: {mesh: drum, scalar: .1}, writable: false}); - - }, undefined, function ( error ) { - console.error( error ); - }); + const modelURL = + "https://samankittani.github.io/GLTF_Model_Server/models/drum/untitled.gltf"; + + loader.load( + modelURL, + function (gltf) { + const drum = gltf.scene.children[0]; + Object.defineProperty(models, "drum", { + value: { mesh: drum, scalar: .1 }, + writable: false, + }); + }, + undefined, + function (error) { + console.error(error); + }, + ); } -function createPianoObj(){ +function createPianoObj() { const loader = new GLTFModule.GLTFLoader(); - const modelURL = 'https://samankittani.github.io/GLTF_Model_Server/models/piano/untitled.gltf'; - - loader.load(modelURL, function ( gltf ) { - const piano = gltf.scene.children[0]; - Object.defineProperty(models, 'piano', {value: {mesh: piano, scalar: 20}, writable: false}); - }, undefined, function ( error ) { - console.error( error ); - }); + const modelURL = + "https://samankittani.github.io/GLTF_Model_Server/models/piano/untitled.gltf"; + + loader.load( + modelURL, + function (gltf) { + const piano = gltf.scene.children[0]; + Object.defineProperty(models, "piano", { + value: { mesh: piano, scalar: 20 }, + writable: false, + }); + }, + undefined, + function (error) { + console.error(error); + }, + ); } -function createBoxObj(hexColor){ - const geometry = new three.BoxGeometry( 1, 1, 1); - const material = new three.MeshPhongMaterial( {color: hexColor} ); +function createBoxObj(hexColor) { + const geometry = new three.BoxGeometry(1, 1, 1); + const material = new three.MeshPhongMaterial({ color: hexColor }); const mesh = new three.Mesh(geometry, material); return mesh; @@ -54,10 +76,10 @@ function createBoxObj(hexColor){ class ThreeJSHandler { constructor() { this.image; - this.context; + this.context; this.renderer; - + this.imageScene; this.imageCamera; this.imagePlane; @@ -72,8 +94,8 @@ class ThreeJSHandler { this.handPositModelSize = 1; // in millimeters this.indexBox; this.handData; - - this.oldIndexBoxPosition = {x: 0, y: 0, z: 0}; + + this.oldIndexBoxPosition = { x: 0, y: 0, z: 0 }; this.handBoxScale = 10; this.posit; @@ -85,56 +107,63 @@ class ThreeJSHandler { this.expFilter = new filters.singleExpFilter([0, 0, 0], .8); this.initScene(); - + this.markers; this.corners; - this.status = 'idle'; + this.status = "idle"; this.expiry = 0; - - }; + } - initScene(){ + initScene() { this.createRenderer(); this.createScenes(); } - createRenderer(){ + createRenderer() { this.renderer = new three.WebGLRenderer(); this.renderer.setClearColor(0xffffff, 1); this.renderer.autoClear = false; } - createScenes(){ + createScenes() { this.imageScene = new three.Scene(); this.imageCamera = new three.OrthographicCamera(-.5, .5, .5, -.5); this.createImagePlane(); this.imageScene.add(this.imageCamera, this.imagePlane); this.objScene = new three.Scene(); - + const stage = world.children[0].stage; - - this.objCamera = new three.PerspectiveCamera(40, stage.width()/stage.height(), 1, 1000); - this.objLight = new three.AmbientLight( 0xffffff); - + + this.objCamera = new three.PerspectiveCamera( + 40, + stage.width() / stage.height(), + 1, + 1000, + ); + this.objLight = new three.AmbientLight(0xffffff); + this.indexBox = createBoxObj(0x5f5f5f); this.objScene.add(this.objCamera, this.objLight, this.indexBox); - } - createImagePlane(){ + createImagePlane() { const texture = new three.CanvasTexture(); const geometry = new three.PlaneGeometry(1.0, 1.0); - const material = new three.MeshBasicMaterial( {map: texture, depthTest: false, depthWrite: false} ); + const material = new three.MeshBasicMaterial({ + map: texture, + depthTest: false, + depthWrite: false, + }); this.imagePlane = new three.Mesh(geometry, material); - - this.imagePlane.position.z = -1; + + this.imagePlane.position.z = -1; } - async getRender(image, obj){ - this.state = 'initializing'; + async getRender(image, obj) { + this.state = "initializing"; this.expiry = +new Date() + 10000; this.markers = await tagModule.getCoordinates(image); @@ -148,26 +177,26 @@ class ThreeJSHandler { return this.render(); } - render(){ + render() { this.renderer.clear(); this.renderer.render(this.imageScene, this.imageCamera); this.renderer.render(this.objScene, this.objCamera); - this.status = 'Idle'; + this.status = "Idle"; return this.renderer.domElement; } - updateValues(image, obj){ - if(!this.image || this.image.width !== image.width){ - this.objCamera.aspect = image.width/image.height; + updateValues(image, obj) { + if (!this.image || this.image.width !== image.width) { + this.objCamera.aspect = image.width / image.height; this.objCamera.updateProjectionMatrix(); this.posit = new POS.Posit(this.positModelSize, image.width); this.handPosit = new POS.Posit(this.handPositModelSize, image.width); this.renderer.setSize(image.width, image.height); } this.image = image; - this.context = this.image.getContext('2d'); - if(this.obj){ + this.context = this.image.getContext("2d"); + if (this.obj) { this.objScene.remove(this.obj); } this.obj = obj.mesh; @@ -175,72 +204,76 @@ class ThreeJSHandler { this.objScale = obj.scalar; this.obj.position.z = -2; - if(this.markers[0]){ + if (this.markers[0]) { this.pose = this.posit.pose(this.markers[0].corners); console.log(this.pose.bestTranslation); this.updateAngles(this.pose.bestRotation); this.moveObj(this.pose.bestTranslation); - } + } } - updateImagePlane(){ + updateImagePlane() { const texture = new three.CanvasTexture(this.image); this.imagePlane.material.map = texture; this.imagePlane.material.needsUpdate = true; } - orientObj(){ - if(this.obj && this.yaw){ + orientObj() { + if (this.obj && this.yaw) { this.obj.rotation.x = -this.pitch; - this.obj.rotation.y = -this.yaw ; - this.obj.rotation.z = this.roll; + this.obj.rotation.y = -this.yaw; + this.obj.rotation.z = this.roll; } } - moveObj(transMatrix){ - if(this.obj){ + moveObj(transMatrix) { + if (this.obj) { this.obj.position.x = transMatrix[0]; this.obj.position.y = transMatrix[1]; this.obj.position.z = -transMatrix[2]; } } - scaleObj(){ - if(this.obj){ + scaleObj() { + if (this.obj) { this.obj.scale.x = this.positModelSize * this.objScale; this.obj.scale.y = this.positModelSize * this.objScale; this.obj.scale.z = this.positModelSize * this.objScale; } } - - updateAngles(matrix){ + + updateAngles(matrix) { this.yaw = Math.atan2(matrix[0][2], matrix[2][2]); this.pitch = Math.asin(-matrix[1][2]); this.roll = Math.atan2(matrix[1][0], matrix[1][1]); - - [this.yaw, this.pitch, this.roll] = this.expFilter.filter(this.yaw, this.pitch, this.roll); + + [this.yaw, this.pitch, this.roll] = this.expFilter.filter( + this.yaw, + this.pitch, + this.roll, + ); } - async updateHandObj(img){ + async updateHandObj(img) { const data = await handModule.findHands(img); - if(!data.landmarks[0]){ + if (!data.landmarks[0]) { return; } this.handData = data.landmarks[0]; - + const corners = this.createCoplanarCoords(this.handData); const pose = this.handPosit.pose(corners); this.transformHandObj(pose); } - transformHandObj(pose){ + transformHandObj(pose) { this.moveHandObj(pose.bestTranslation); this.scaleHandObj(); } - moveHandObj(transMatrix){ + moveHandObj(transMatrix) { console.log(transMatrix); - if(this.obj){ + if (this.obj) { this.oldIndexBoxPosition.x = this.indexBox.position.x; this.oldIndexBoxPosition.y = this.indexBox.position.y; this.oldIndexBoxPosition.z = this.indexBox.position.z; @@ -251,65 +284,81 @@ class ThreeJSHandler { } } - scaleHandObj(){ - if(this.obj){ + scaleHandObj() { + if (this.obj) { this.indexBox.scale.x = this.handPositModelSize * this.handBoxScale; this.indexBox.scale.y = this.handPositModelSize * this.handBoxScale; this.indexBox.scale.z = this.handPositModelSize * this.handBoxScale; } } - createCoplanarCoords(data){ + createCoplanarCoords(data) { const centeredData = this.centerData(data); const res = new Array(); const sep = this.getSeperation() * 3; - - res.push({x: Math.floor(centeredData[8].x) + sep, y: Math.floor(centeredData[8].y) - sep}) - res.push({x: Math.floor(centeredData[8].x) - sep, y: Math.floor(centeredData[8].y) - sep}) - res.push({x: Math.floor(centeredData[8].x) - sep, y: Math.floor(centeredData[8].y) + sep}) - res.push({x: Math.floor(centeredData[8].x) + sep, y: Math.floor(centeredData[8].y) + sep}) + + res.push({ + x: Math.floor(centeredData[8].x) + sep, + y: Math.floor(centeredData[8].y) - sep, + }); + res.push({ + x: Math.floor(centeredData[8].x) - sep, + y: Math.floor(centeredData[8].y) - sep, + }); + res.push({ + x: Math.floor(centeredData[8].x) - sep, + y: Math.floor(centeredData[8].y) + sep, + }); + res.push({ + x: Math.floor(centeredData[8].x) + sep, + y: Math.floor(centeredData[8].y) + sep, + }); return res; } - centerData(data){ + centerData(data) { const res = new Array(); - for(const mark of data){ + for (const mark of data) { const newData = { - x: (mark.x * this.image.width) - (this.image.width/2), - y: (this.image.height/2) - ((mark.y * this.image.height))} + x: (mark.x * this.image.width) - (this.image.width / 2), + y: (this.image.height / 2) - (mark.y * this.image.height), + }; res.push(newData); } return res; } - getSeperation(){ + getSeperation() { const data = this.handData; - if(!data || data.length === 0) { + if (!data || data.length === 0) { return; } - const dist = Math.sqrt(((data[8].x - data[6].x))**2 + - ((data[8].y - data[6].y))**2 + - ((data[8].z - data[6].z) * 2)**2); - - console.log(`dist: ${dist}`) + const dist = Math.sqrt( + (data[8].x - data[6].x) ** 2 + + (data[8].y - data[6].y) ** 2 + + ((data[8].z - data[6].z) * 2) ** 2, + ); + + console.log(`dist: ${dist}`); return dist; } - intersectionForce(){ + intersectionForce() { const eucDist = Math.sqrt( - (this.indexBox.position.x - this.oldIndexBoxPosition.x) ** 2 + - (this.indexBox.position.y - this.oldIndexBoxPosition.y) ** 2 + - ((this.indexBox.position.z - this.oldIndexBoxPosition.z)) ** 2) - - const speed = this.isHandOverlapping()? eucDist: 0; + (this.indexBox.position.x - this.oldIndexBoxPosition.x) ** 2 + + (this.indexBox.position.y - this.oldIndexBoxPosition.y) ** 2 + + (this.indexBox.position.z - this.oldIndexBoxPosition.z) ** 2, + ); + + const speed = this.isHandOverlapping() ? eucDist : 0; return speed; } - isHandOverlapping(){ + isHandOverlapping() { let objMesh = this.obj; - while(!objMesh.isMesh){ + while (!objMesh.isMesh) { objMesh = objMesh.children[0]; } objMesh.updateMatrixWorld(); @@ -319,17 +368,18 @@ class ThreeJSHandler { let bounding2 = this.indexBox.geometry.boundingBox.clone(); bounding2.applyMatrix4(this.indexBox.matrixWorld); - if(bounding1.intersectsBox(bounding2)){ + if (bounding1.intersectsBox(bounding2)) { return true; } return false; } isIdle() { - if(this.status === 'Idle') + if (this.status === "Idle") { return true; - if(+new Date() > this.expiry){ - this.state = 'Idle'; + } + if (+new Date() > this.expiry) { + this.state = "Idle"; return true; } return false; @@ -337,21 +387,21 @@ class ThreeJSHandler { } const RENDERERS = []; -function getRenderer(){ - if(!three) {throw new Error('three not loaded yet')} - for(const renderer of RENDERERS){ - if(renderer.isIdle()){ +function getRenderer() { + if (!three) throw new Error("three not loaded yet"); + for (const renderer of RENDERERS) { + if (renderer.isIdle()) { return renderer; } } - const newRenderer = new ThreeJSHandler() + const newRenderer = new ThreeJSHandler(); RENDERERS.push(newRenderer); return newRenderer; } -async function renderScene(image, objKey){ +async function renderScene(image, objKey) { const renderer = getRenderer(); return await renderer.getRender(image, models[objKey]); } -export {renderScene} \ No newline at end of file +export { renderScene }; diff --git a/extensions/AugmentedReality/js/tagHandler.mjs b/extensions/AugmentedReality/js/tagHandler.mjs index 972511b..59b5b22 100644 --- a/extensions/AugmentedReality/js/tagHandler.mjs +++ b/extensions/AugmentedReality/js/tagHandler.mjs @@ -2,27 +2,31 @@ const positVersion = 1; class aprilTagHandler { constructor() { - this.detector = new AR.Detector({dictionaryName: aprilTagHandler.config.dictionary, maxHammingDistance: 3}); - this.state = 'Idle'; + this.detector = new AR.Detector({ + dictionaryName: aprilTagHandler.config.dictionary, + maxHammingDistance: 3, + }); + this.state = "Idle"; this.expiry = 0; } static config = { - dictionary: 'APRILTAG_16h5' - } + dictionary: "APRILTAG_16h5", + }; - async detectAR(imageData){ - this.state = 'Detecting...'; + async detectAR(imageData) { + this.state = "Detecting..."; this.expiry = +new Date() + 10000; const markers = this.detector.detect(imageData); - this.state = 'Idle'; + this.state = "Idle"; return markers; } isIdle() { - if(this.status === 'Idle') + if (this.status === "Idle") { return true; - if(+new Date() > this.expiry){ - this.state = 'Idle'; + } + if (+new Date() > this.expiry) { + this.state = "Idle"; return true; } return false; @@ -31,9 +35,9 @@ class aprilTagHandler { const APRILTAGHANDLERS = []; -function getAprilHandler(){ - for(const handler of APRILTAGHANDLERS){ - if(handler.isIdle()){ +function getAprilHandler() { + for (const handler of APRILTAGHANDLERS) { + if (handler.isIdle()) { return handler; } } @@ -42,44 +46,43 @@ function getAprilHandler(){ return newHandler; } -function resetAllDetectors(dict){ - - for(const handler of APRILTAGHANDLERS){ +function resetAllDetectors(dict) { + for (const handler of APRILTAGHANDLERS) { (async (handler, dict) => { console.log(handler, dict); - handler.detector = new AR.Detector({dictionaryName: dict}); + handler.detector = new AR.Detector({ dictionaryName: dict }); console.log(handler, dict); })(handler, dict); } } -async function getCoordinates(image){ +async function getCoordinates(image) { const handler = getAprilHandler(); - const context = image.getContext('2d'); - const imageData = context.getImageData(0,0,image.width, image.height); - + const context = image.getContext("2d"); + const imageData = context.getImageData(0, 0, image.width, image.height); + const markers = await handler.detectAR(imageData); return markers; } -function transformCoordinates(markers, image){ - if(markers.length === 0){ +function transformCoordinates(markers, image) { + if (markers.length === 0) { return markers; } - for(const marker of markers){ - for(const corner of marker.corners){ - corner.x = corner.x - (image.width/2); - corner.y = 1 - (corner.y - (image.height/2)); + for (const marker of markers) { + for (const corner of marker.corners) { + corner.x = corner.x - (image.width / 2); + corner.y = 1 - (corner.y - (image.height / 2)); } - return markers; + return markers; } } async function isTagVisible(image, value) { const markers = await getCoordinates(image); - for(const marker of markers){ - if(value.indexOf(marker.id) !== -1){ + for (const marker of markers) { + if (value.indexOf(marker.id) !== -1) { return true; } } @@ -88,20 +91,22 @@ async function isTagVisible(image, value) { async function getVisibleTags(image) { const handler = getAprilHandler(); - const res = (new Array(handler.detector.dictionary.codeList.length)).fill(false); + const res = (new Array(handler.detector.dictionary.codeList.length)).fill( + false, + ); const markers = await getCoordinates(image); - for(const marker of markers){ + for (const marker of markers) { res[marker.id] = true; } return res; } -function getDictionary(){ +function getDictionary() { return aprilTagHandler.config.dictionary; } -function setDictionary(dict){ - if(dict === aprilTagHandler.config.dictionary){ +function setDictionary(dict) { + if (dict === aprilTagHandler.config.dictionary) { return; } aprilTagHandler.config.dictionary = dict; @@ -109,33 +114,37 @@ function setDictionary(dict){ } const DEVURL = { - SVDURL: - 'https://samankittani.github.io/js-aruco2/src/svd.js', - POSIT1URL: + SVDURL: "https://samankittani.github.io/js-aruco2/src/svd.js", + POSIT1URL: `https://samankittani.github.io/js-aruco2/src/posit${positVersion}.js`, - CVURL: - 'https://samankittani.github.io/js-aruco2/src/cv.js', - ARUCOURL: - 'https://samankittani.github.io/js-aruco2/src/aruco.js', - TAGDICURL01: - 'https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5.js', - TAGDICURL02: - 'https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5_duo.js', - TAGDICURL03: - 'https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5_mini.js' - } - const sources = Object.values(DEVURL); - - if((document.getElementsByClassName("ARScripts")).length === 0){ - for(const source of sources){ - const script = document.createElement('script'); - script.className = 'ARScipts'; - script.type = 'text/javascript'; - script.src = source; - script.async = false; - script.crossOrigin = 'anonymous'; - document.body.appendChild(script); - } - } + CVURL: "https://samankittani.github.io/js-aruco2/src/cv.js", + ARUCOURL: "https://samankittani.github.io/js-aruco2/src/aruco.js", + TAGDICURL01: + "https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5.js", + TAGDICURL02: + "https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5_duo.js", + TAGDICURL03: + "https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5_mini.js", +}; +const sources = Object.values(DEVURL); + +if ((document.getElementsByClassName("ARScripts")).length === 0) { + for (const source of sources) { + const script = document.createElement("script"); + script.className = "ARScipts"; + script.type = "text/javascript"; + script.src = source; + script.async = false; + script.crossOrigin = "anonymous"; + document.body.appendChild(script); + } +} -export {getCoordinates, transformCoordinates, isTagVisible, getVisibleTags, getDictionary, setDictionary} +export { + getCoordinates, + getDictionary, + getVisibleTags, + isTagVisible, + setDictionary, + transformCoordinates, +}; diff --git a/extensions/AugmentedReality/js/ui-morphs.js b/extensions/AugmentedReality/js/ui-morphs.js index bf9393f..044cf08 100644 --- a/extensions/AugmentedReality/js/ui-morphs.js +++ b/extensions/AugmentedReality/js/ui-morphs.js @@ -1,7 +1,9 @@ -const localhost = window.location.search.includes('localhost'); -const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/'; +const localhost = window.location.search.includes("localhost"); +const root = localhost + ? "http://localhost:8000/" + : "https://extensions.netsblox.org/"; -function ArucoGenMorph(){ +function ArucoGenMorph() { this.init(); } @@ -12,45 +14,53 @@ ArucoGenMorph.uber = DialogBoxMorph.prototype; ArucoGenMorph.prototype.init = function () { ArucoGenMorph.uber.init.call(this); - this.labelString = 'AprilTag Code Generator'; + this.labelString = "AprilTag Code Generator"; this.createLabel(); const width = 300; const height = 300; this.bounds.setWidth(width); this.bounds.setHeight(height); - this.dictionary = new AR.Dictionary('APRILTAG_16h5'); - - this.add(this.IDPrompt = new StringMorph('Code Value: [0-29]')); - this.add(this.saveButton = new PushButtonMorph(null, () => this.download(), 'Download')); - this.add(this.closeButton = new PushButtonMorph(null, () => this.destroy(), 'Close')); + this.dictionary = new AR.Dictionary("APRILTAG_16h5"); + + this.add(this.IDPrompt = new StringMorph("Code Value: [0-29]")); + this.add( + this.saveButton = new PushButtonMorph( + null, + () => this.download(), + "Download", + ), + ); + this.add( + this.closeButton = new PushButtonMorph(null, () => this.destroy(), "Close"), + ); const [qrwidth, qrheight] = [150, 150]; this.add(this.ARCode = new Morph()); this.ARCode.setWidth(qrwidth); this.ARCode.setHeight(qrheight); - this.ARCode.value = '0'; - - const options = Array.from(Array(30).keys()) - for(item of options){ + this.ARCode.value = "0"; + + const options = Array.from(Array(30).keys()); + for (item of options) { options[item] = item.toString(); } const menuOptions = Object.assign({}, options); - this.add(this.IDInput = new InputFieldMorph('0', true, menuOptions, true)); - + this.add(this.IDInput = new InputFieldMorph("0", true, menuOptions, true)); + this.fixLayout(); this.rerender(); }; ArucoGenMorph.prototype.download = async function () { - console.log('downloading attempt'); - if(this.ARCode.texture){ + console.log("downloading attempt"); + if (this.ARCode.texture) { const svg = await (await fetch(this.ARCode.texture)).blob(); console.log(svg); const url = window.URL.createObjectURL(svg); - const name = `AprilTag_16h5_ID${this.ARCode.value}.svg` - const link = document.createElement('a'); + const name = `AprilTag_16h5_ID${this.ARCode.value}.svg`; + const link = document.createElement("a"); link.download = name; link.href = url; document.body.appendChild(link); @@ -58,26 +68,27 @@ ArucoGenMorph.prototype.download = async function () { document.body.removeChild(link); window.URL.revokeObjectURL(url); } -} +}; ArucoGenMorph.prototype.updateARCode = function () { const newValue = parseInt(this.IDInput.children[0].children[0].text); - if(newValue > 29 || isNaN(newValue)){ + if (newValue > 29 || isNaN(newValue)) { return; } - this.ARCode.value = newValue; - console.log('value: ', this.ARCode.value); - + this.ARCode.value = newValue; + console.log("value: ", this.ARCode.value); + this.ARCode.cachedTexture = null; - this.ARCode.texture = root + `extensions/AugmentedReality/tag_svgs/apriltag_16h5_svgs/APRILTAG_16h5_ID${this.ARCode.value}.svg`; + this.ARCode.texture = root + + `extensions/AugmentedReality/tag_svgs/apriltag_16h5_svgs/APRILTAG_16h5_ID${this.ARCode.value}.svg`; this.ARCode.rerender(); -} +}; ArucoGenMorph.prototype.reactToChoice = function () { this.updateARCode(); -} +}; -ArucoGenMorph.prototype.fixLayout = function() { +ArucoGenMorph.prototype.fixLayout = function () { ArucoGenMorph.uber.fixLayout.call(this); if (this.IDPrompt) { @@ -91,7 +102,6 @@ ArucoGenMorph.prototype.fixLayout = function() { this.IDInput.bounds.setWidth(30); this.IDInput.children[0].bounds.setWidth(25); this.IDInput.children[1].setLeft(this.IDInput.left() + 15); - } if (this.ARCode) { this.ARCode.setCenter(this.center()); @@ -111,107 +121,14 @@ ArucoGenMorph.prototype.fixLayout = function() { ArucoGenMorph.prototype.accept = function () { if (this.action) { - if (typeof this.target === 'function') { - if (typeof this.action === 'function') { - this.target.call(this.environment, this.action.call()); - } else { - this.target.call(this.environment, this.action); - } - } else { - if (typeof this.action === 'function') { - this.action.call(this.target, this.getInput()); - } else { // assume it's a String - this.target[this.action](this.getInput()); - } - } - } - this.updateARCode(); -}; - -/* In development - -function DrumSetMorph (){ - this.init(); -} - -DrumSetMorph.prototype = new DialogBoxMorph(); -DrumSetMorph.prototype.constructor = DrumSetMorph; -DrumSetMorph.uber = DialogBoxMorph.prototype; - -DrumSetMorph.prototype.init = function () { - DrumSetMorph.uber.init.call(this); - - this.labelString = 'Augmented Reality Instrument'; - this.createLabel(); - - const width = 300; - const height = 300; - this.bounds.setWidth(width); - this.bounds.setHeight(height); - - this.add(this.closeButton = new PushButtonMorph(null, () => this.destroy(), 'Close')); - - const [imgWidth, imgHeight] = [150, 150]; - this.add(this.ARCode = new Morph()); - this.ARCode.setWidth(imgWidth); - this.ARCode.setHeight(imgHeight); - - const menuOptions = ['DrumSet']; - - this.add(this.IDPrompt = new StringMorph('Instrument: ')); - this.add(this.IDInput = new InputFieldMorph('0', true, menuOptions, true)); - - this.fixLayout(); - this.rerender(); -}; - -DrumSetMorph.prototype.download = async function () { - -} - -DrumSetMorph.prototype.updateARCode = function () { - -} - -DrumSetMorph.prototype.reactToChoice = function () { - -} - -DrumSetMorph.prototype.fixLayout = function() { - DrumSetMorph.uber.fixLayout.call(this); - - if (this.IDPrompt) { - this.IDPrompt.setCenter(this.center()); - this.IDPrompt.setTop(35); - } - if (this.IDInput) { - this.IDInput.setCenter(this.center()); - this.IDInput.setTop(50); - this.IDInput.setLeft(135); - this.IDInput.bounds.setWidth(30); - this.IDInput.children[0].bounds.setWidth(25); - this.IDInput.children[1].setLeft(this.IDInput.left() + 15); - } - if (this.ARCode) { - this.ARCode.setCenter(this.center()); - this.ARCode.setTop(90); - } - if (this.closeButton) { - this.closeButton.setCenter(this.center()); - this.closeButton.setBottom(this.bottom() - 20); - } -}; - -DrumSetMorph.prototype.accept = function () { - if (this.action) { - if (typeof this.target === 'function') { - if (typeof this.action === 'function') { + if (typeof this.target === "function") { + if (typeof this.action === "function") { this.target.call(this.environment, this.action.call()); } else { this.target.call(this.environment, this.action); } } else { - if (typeof this.action === 'function') { + if (typeof this.action === "function") { this.action.call(this.target, this.getInput()); } else { // assume it's a String this.target[this.action](this.getInput()); @@ -220,5 +137,3 @@ DrumSetMorph.prototype.accept = function () { } this.updateARCode(); }; - - */ \ No newline at end of file diff --git a/extensions/FaceLandmarker/index.js b/extensions/FaceLandmarker/index.js index 79c5bed..5944ab5 100644 --- a/extensions/FaceLandmarker/index.js +++ b/extensions/FaceLandmarker/index.js @@ -1,17 +1,18 @@ (async function () { - - const localhost = window.location.search.includes('localhost'); - const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/'; - + const localhost = window.location.search.includes("localhost"); + const root = localhost + ? "http://localhost:8000/" + : "https://extensions.netsblox.org/"; + const DEVURL = { - MODELPATHURL: 'https://samankittani.github.io/mediapipe_models/models/face_landmarker.task', - VISIONMODULELOADERURL: root + 'utils/visionModuleLoader.js' - } + MODELPATHURL: + "https://samankittani.github.io/mediapipe_models/models/face_landmarker.task", + VISIONMODULELOADERURL: root + "utils/visionModuleLoader.js", + }; class FaceHandler { constructor() { - - this.faceLandmarker = null; + this.faceLandmarker = null; this.expiry = 0; this.resolve = null; @@ -27,47 +28,53 @@ minFaceDetConf: .5, minFacePresConf: .5, minTracConf: .5, - segMask: false - } - } + segMask: false, + }, + }; async generateTask() { - if(!Vision) - throw Error('Vision Module is loading...'); - - if(this.faceLandmarker === 'loading...') - throw Error('faceLandmarker is currently loading'); - - this.faceLandmarker = 'loading...'; - - this.faceLandmarker = await Vision.Module.FaceLandmarker.createFromOptions(Vision.Task, { - baseOptions: { - modelAssetPath: DEVURL.MODELPATHURL, - delegate: 'GPU' - }, - runningMode: 'VIDEO', - outputFaceBlendshapes: true, - numFaces: FaceHandler.config.options.numFaces, - minFaceDetectionConfidence: FaceHandler.config.options.minFaceDetConf, - minFacePresenceConfidence: FaceHandler.config.options.minFacePresConf, - minTrackingConfidence: FaceHandler.config.options.minTracConf - }).catch((err) => { - throw new Error(`Failed to generate vision task: Error: ${err}`); - }) + if (!Vision) { + throw Error("Vision Module is loading..."); + } + + if (this.faceLandmarker === "loading...") { + throw Error("faceLandmarker is currently loading"); + } + + this.faceLandmarker = "loading..."; + + this.faceLandmarker = await Vision.Module.FaceLandmarker + .createFromOptions(Vision.Task, { + baseOptions: { + modelAssetPath: DEVURL.MODELPATHURL, + delegate: "GPU", + }, + runningMode: "VIDEO", + outputFaceBlendshapes: true, + numFaces: FaceHandler.config.options.numFaces, + minFaceDetectionConfidence: FaceHandler.config.options.minFaceDetConf, + minFacePresenceConfidence: FaceHandler.config.options.minFacePresConf, + minTrackingConfidence: FaceHandler.config.options.minTracConf, + }).catch((err) => { + throw new Error(`Failed to generate vision task: Error: ${err}`); + }); } - async infer(image){ - if(this.faceLandmarker === null){ + async infer(image) { + if (this.faceLandmarker === null) { await this.generateTask(); } - if(this.faceLandmarker === 'loading...') { - throw new Error('handLandmarker is loading...') + if (this.faceLandmarker === "loading...") { + throw new Error("handLandmarker is loading..."); } - if(this.resolve !== null) throw Error('FaceHandler is currently in use'); - this.resolve = 'loading...'; + if (this.resolve !== null) throw Error("FaceHandler is currently in use"); + this.resolve = "loading..."; this.frameTime = performance.now(); - if(FaceHandler.config.data !== null && ((this.frameTime - FaceHandler.config.updateTime) < 10)){ + if ( + FaceHandler.config.data !== null && + ((this.frameTime - FaceHandler.config.updateTime) < 10) + ) { this.resolve = null; return FaceHandler.config.data; } @@ -75,11 +82,14 @@ this.expiry = +new Date() + 10000; - return new Promise(resolve => { - FaceHandler.config.data = this.faceLandmarker.detectForVideo(image, this.frameTime); + return new Promise((resolve) => { + FaceHandler.config.data = this.faceLandmarker.detectForVideo( + image, + this.frameTime, + ); this.resolve = null; resolve(FaceHandler.config.data); - }) + }); } isIdle() { @@ -90,13 +100,14 @@ } return false; } - } + } const FACE_HANDLES = []; function getFaceHandler() { for (const handle of FACE_HANDLES) { - if (handle.isIdle()) + if (handle.isIdle()) { return handle; + } } const handle = new FaceHandler(); FACE_HANDLES.push(handle); @@ -110,57 +121,57 @@ async function renderFace(image) { const data = await findFace(image); - if(typeof(data) === 'string'){ + if (typeof data === "string") { return data; } - const context = image.getContext('2d'); + const context = image.getContext("2d"); const drawer = new Vision.Module.DrawingUtils(context); for (const landmarks of data.faceLandmarks) { drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_TESSELATION, - { color: '#C0C0C070', lineWidth: 1 } + { color: "#C0C0C070", lineWidth: 1 }, ); drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE, - { color: '#30FF30' } + { color: "#30FF30" }, ); drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW, - { color: '#30FF30' } + { color: "#30FF30" }, ); drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_LEFT_EYE, - { color: '#30FF30' } + { color: "#30FF30" }, ); drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW, - { color: '#30FF30' } + { color: "#30FF30" }, ); drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_FACE_OVAL, - { color: '#30FF30' } + { color: "#30FF30" }, ); drawer.drawConnectors( - landmarks, - Vision.Module.FaceLandmarker.FACE_LANDMARKS_LIPS, - { color: '#30FF30' } + landmarks, + Vision.Module.FaceLandmarker.FACE_LANDMARKS_LIPS, + { color: "#30FF30" }, ); drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS, - { color: '#30FF30' } + { color: "#30FF30" }, ); drawer.drawConnectors( landmarks, Vision.Module.FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS, - { color: '#30FF30' } + { color: "#30FF30" }, ); } return image; @@ -171,60 +182,78 @@ const res = []; for (const item of value) res.push(snapify(item)); return new List(res); - } else if (typeof(value) === 'object') { + } else if (typeof value === "object") { const res = []; for (const key in value) res.push(new List([key, snapify(value[key])])); return new List(res); } else return value; } - + class FaceLandmarker extends Extension { constructor(ide) { - super('FaceLandmarker'); + super("FaceLandmarker"); this.ide = ide; } onOpenRole() {} - getMenu() { return {}; } + getMenu() { + return {}; + } - getCategories() { return []; } + getCategories() { + return []; + } getPalette() { const blocks = [ - new Extension.Palette.Block('faceLandmarksFindFace'), - new Extension.Palette.Block('faceLandmarksRender'), + new Extension.Palette.Block("faceLandmarksFindFace"), + new Extension.Palette.Block("faceLandmarksRender"), ]; return [ - new Extension.PaletteCategory('sensing', blocks, SpriteMorph), - new Extension.PaletteCategory('sensing', blocks, StageMorph), + new Extension.PaletteCategory("sensing", blocks, SpriteMorph), + new Extension.PaletteCategory("sensing", blocks, StageMorph), ]; } getBlocks() { function block(name, type, category, spec, defaults, action) { - return new Extension.Block(name, type, category, spec, defaults, action).for(SpriteMorph, StageMorph) + return new Extension.Block(name, type, category, spec, defaults, action) + .for(SpriteMorph, StageMorph); } return [ - block('faceLandmarksFindFace', 'reporter', 'sensing', 'get face data from %s', [], function (img) { - return this.runAsyncFn(async () => { - img = img?.contents || img; - if (!img || typeof(img) !== 'object' || !img.width || !img.height) throw Error('Expected an image as input'); + block( + "faceLandmarksFindFace", + "reporter", + "sensing", + "get face data from %s", + [], + function (img) { + return this.runAsyncFn(async () => { + img = img?.contents || img; + if ( + !img || typeof img !== "object" || !img.width || !img.height + ) throw Error("Expected an image as input"); - const res = await findFace(img); - - return snapify(res); - }, { args: [], timeout: 10000 }); - }), - - block('faceLandmarksRender', 'reporter', 'sensing', 'render face %s', [''], function (img) { + const res = await findFace(img); + + return snapify(res); + }, { args: [], timeout: 10000 }); + }, + ), + + block("faceLandmarksRender", "reporter", "sensing", "render face %s", [ + "", + ], function (img) { return this.runAsyncFn(async () => { img = img?.contents || img; - - if (!img || typeof(img) !== 'object' || !img.width || !img.height) {throw Error('Expected an image as input');} - + + if (!img || typeof img !== "object" || !img.width || !img.height) { + throw Error("Expected an image as input"); + } + const res = await renderFace(img); - + return new Costume(res); }, { args: [], timeout: 10000 }); }), @@ -237,19 +266,18 @@ for (const x of s) res[x] = x; return res; } - return [ - ]; + return []; } } const visionSource = DEVURL.VISIONMODULELOADERURL; - if(!document.getElementById('visionModule')){ - const script = document.createElement('script'); - script.id = 'visionModule'; - script.type = 'text/javascript'; + if (!document.getElementById("visionModule")) { + const script = document.createElement("script"); + script.id = "visionModule"; + script.type = "text/javascript"; script.src = visionSource; script.async = false; - script.crossOrigin = 'anonymous'; + script.crossOrigin = "anonymous"; document.body.appendChild(script); } diff --git a/extensions/HandGestures/handLandmarkerModule.mjs b/extensions/HandGestures/handLandmarkerModule.mjs index 234a817..f2b0432 100644 --- a/extensions/HandGestures/handLandmarkerModule.mjs +++ b/extensions/HandGestures/handLandmarkerModule.mjs @@ -1,46 +1,65 @@ - -const localhost = window.location.search.includes('localhost'); -const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/'; +const localhost = window.location.search.includes("localhost"); +const root = localhost + ? "http://localhost:8000/" + : "https://extensions.netsblox.org/"; const DEVURL = { - MODELPATHURL: 'https://samankittani.github.io/mediapipe_models/models/hand_landmarker.task', - VISIONMODULELOADERURL: root + 'utils/visionModuleLoader.js' -} - -const LANDMARKS = ['WRIST','THUMB_cmc','THUMB_mcp','THUMB_ip','THUMB_tip', - 'INDEX_mcp','INDEX_pip','INDEX_dip','INDEX_tip', - 'MIDDLE_mcp','MIDDLE_pip','MIDDLE_dip','MIDDLE_tip', - 'RING_mcp','RING_pip','RING_dip','RING_tip', - 'PINKY_mcp','PINKY_pip','PINKY_dip','PINKY_tip']; - -const DATAOPTIONS = ['Hand Landmarks', - 'World Landmarks', - 'Handedness' ]; - -const CONFIGOPTIONS = ['Min Detect Confidence', - 'Min Presence Confidence', - 'Min Track Confidence', - 'Max Hands' ]; - + MODELPATHURL: + "https://samankittani.github.io/mediapipe_models/models/hand_landmarker.task", + VISIONMODULELOADERURL: root + "utils/visionModuleLoader.js", +}; + +const LANDMARKS = [ + "WRIST", + "THUMB_cmc", + "THUMB_mcp", + "THUMB_ip", + "THUMB_tip", + "INDEX_mcp", + "INDEX_pip", + "INDEX_dip", + "INDEX_tip", + "MIDDLE_mcp", + "MIDDLE_pip", + "MIDDLE_dip", + "MIDDLE_tip", + "RING_mcp", + "RING_pip", + "RING_dip", + "RING_tip", + "PINKY_mcp", + "PINKY_pip", + "PINKY_dip", + "PINKY_tip", +]; + +const DATAOPTIONS = ["Hand Landmarks", "World Landmarks", "Handedness"]; + +const CONFIGOPTIONS = [ + "Min Detect Confidence", + "Min Presence Confidence", + "Min Track Confidence", + "Max Hands", +]; /* comparison helper */ -function inRange(x, min, max){ - return min <= x && x <= max; +function inRange(x, min, max) { + return min <= x && x <= max; } /* dataOptionConverter */ -function convertOption(option){ +function convertOption(option) { const index = DATAOPTIONS.indexOf(option); switch (index) { case 0: - return 'landmarks'; + return "landmarks"; case 1: - return 'worldLandmarks'; - + return "worldLandmarks"; + case 2: - return 'handedness'; - + return "handedness"; + default: return undefined; } @@ -48,8 +67,7 @@ function convertOption(option){ class VisionHandler { constructor() { - - this.modelPathUrl = DEVURL.MODELPATHURL; + this.modelPathUrl = DEVURL.MODELPATHURL; this.handLandmarker = null; this.expiry = 0; @@ -65,44 +83,51 @@ class VisionHandler { minDetConf: .5, minPresConf: .5, minTracConf: .5, - maxHands: 2 - } - } + maxHands: 2, + }, + }; async generateTask() { - if(!Vision) - throw Error('Vision Module is loading'); - - this.handLandmarker = 'loading...'; - console.log('Vision:', Vision.Task); - this.handLandmarker = await Vision.Module.HandLandmarker.createFromOptions(Vision.Task, { - baseOptions: { - modelAssetPath: this.modelPathUrl, - delegate: 'GPU' + if (!Vision) { + throw Error("Vision Module is loading"); + } + + this.handLandmarker = "loading..."; + console.log("Vision:", Vision.Task); + this.handLandmarker = await Vision.Module.HandLandmarker.createFromOptions( + Vision.Task, + { + baseOptions: { + modelAssetPath: this.modelPathUrl, + delegate: "GPU", + }, + numHands: VisionHandler.config.options.maxHands, + runningMode: "VIDEO", + minHandDetectionConfidence: VisionHandler.config.options.minDetConf, + minHandPresenceConfidence: VisionHandler.config.options.minPresConf, + minTrackingConfidence: VisionHandler.config.options.minTracConf, }, - numHands: VisionHandler.config.options.maxHands, - runningMode: 'VIDEO', - minHandDetectionConfidence: VisionHandler.config.options.minDetConf, - minHandPresenceConfidence: VisionHandler.config.options.minPresConf, - minTrackingConfidence: VisionHandler.config.options.minTracConf - }).catch((err) => { + ).catch((err) => { this.handLandmarker = null; throw new Error(`Failed to generate vision task, Error: ${err}`); - }) + }); } - async infer(image){ - if(this.handLandmarker === null) { + async infer(image) { + if (this.handLandmarker === null) { await this.generateTask(); } - if(this.handLandmarker === 'loading...') { - throw new Error('handLandmarker is loading...'); + if (this.handLandmarker === "loading...") { + throw new Error("handLandmarker is loading..."); } - if(this.resolve !== null) throw Error('VisionHandler is currently in use'); - this.resolve = 'loading...'; + if (this.resolve !== null) throw Error("VisionHandler is currently in use"); + this.resolve = "loading..."; this.frameTime = performance.now(); - if(VisionHandler.config.data !== null && ((this.frameTime - VisionHandler.config.updateTime) < 5)){ + if ( + VisionHandler.config.data !== null && + ((this.frameTime - VisionHandler.config.updateTime) < 5) + ) { this.resolve = null; return VisionHandler.config.data; } @@ -110,11 +135,14 @@ class VisionHandler { this.expiry = +new Date() + 10000; - return new Promise(resolve => { - VisionHandler.config.data = this.handLandmarker.detectForVideo(image, this.frameTime); + return new Promise((resolve) => { + VisionHandler.config.data = this.handLandmarker.detectForVideo( + image, + this.frameTime, + ); this.resolve = null; resolve(VisionHandler.config.data); - }) + }); } isIdle() { @@ -126,23 +154,24 @@ class VisionHandler { return false; } - // This function takes new model parameters and sets them for all handlers + // This function takes new model parameters and sets them for all handlers async resetHandleOptions() { await this.handLandmarker.setOptions({ minHandPresenceConfidence: VisionHandler.config.options.minPresConf, minHandDetectionConfidence: VisionHandler.config.options.minDetConf, minTrackingConfidence: VisionHandler.config.options.minTracConf, - numHands: VisionHandler.config.options.maxHands + numHands: VisionHandler.config.options.maxHands, }); console.log(this.handLandmarker); } -} +} const VISION_HANDLES = []; function getVisionHandler() { for (const handle of VISION_HANDLES) { - if (handle.isIdle()) + if (handle.isIdle()) { return handle; + } } const handle = new VisionHandler(); VISION_HANDLES.push(handle); @@ -157,60 +186,64 @@ async function findHands(image) { async function renderHands(image) { const data = await findHands(image); - const context = image.getContext('2d'); + const context = image.getContext("2d"); const drawer = new Vision.Module.DrawingUtils(context); - + for (const landmarks of data.landmarks) { - drawer.drawConnectors(landmarks, Vision.Module.HandLandmarker.HAND_CONNECTIONS, { color: '#00ff00', lineWidth: 3 }); - drawer.drawLandmarks(landmarks, { color: '#ff0000', lineWidth: 1 }); + drawer.drawConnectors( + landmarks, + Vision.Module.HandLandmarker.HAND_CONNECTIONS, + { color: "#00ff00", lineWidth: 3 }, + ); + drawer.drawLandmarks(landmarks, { color: "#ff0000", lineWidth: 1 }); } return image; } -async function getCentralCoords(image){ +async function getCentralCoords(image) { const data = await findHands(image); - if(!data || data.landmarks.length){ + if (!data || data.landmarks.length) { return data; } const imageCoords = data.landmarks; - for(const hands of imageCoords){ - for(const coord of hands){ - coord.x = (coord.x * image.width) - (image.width/2); - coord.y = 1 - ((coord.y * image.height) - (image.height/2)); - coord.z = (coord.z * image.width) - (image.width/2); + for (const hands of imageCoords) { + for (const coord of hands) { + coord.x = (coord.x * image.width) - (image.width / 2); + coord.y = 1 - ((coord.y * image.height) - (image.height / 2)); + coord.z = (coord.z * image.width) - (image.width / 2); } } return imageCoords; } -async function getCentralCoord(image, landmark){ +async function getCentralCoord(image, landmark) { const data = await findHands(image); - if(!data || !data.landmarks.length){ + if (!data || !data.landmarks.length) { return; } - const index = LANDMARKS.indexOf(landmark); - const coords = data.landmarks[0][index]; /* does hand 0 everytime */ - + const index = LANDMARKS.indexOf(landmark); + const coords = data.landmarks[0][index]; /* does hand 0 everytime */ + const centralCoords = { - x: (coords.x * image.width) - (image.width/2), - y: 1 - (coords.y * image.height) - (image.height/2), - z: (coords.z) //* image.width) - (image.width/2) - } + x: (coords.x * image.width) - (image.width / 2), + y: 1 - (coords.y * image.height) - (image.height / 2), + z: (coords.z), //* image.width) - (image.width/2) + }; return centralCoords; } -async function parseLandmark(image, option, landmark) { +async function parseLandmark(image, option, landmark) { const data = await findHands(image); const convOption = convertOption(option); const coords = data[convOption]; const res = new Array(); - - for(const hand of coords){ + + for (const hand of coords) { const index = LANDMARKS.indexOf(landmark); - res.push(hand[index]); - } + res.push(hand[index]); + } return res; } @@ -218,14 +251,17 @@ async function parseLandmarkDistance(image, landmark1, landmark2) { const data = await findHands(image); const coords = data.landmarks; const res = new Array(); - - for(const hand of coords){ + + for (const hand of coords) { const index1 = LANDMARKS.indexOf(landmark1); const index2 = LANDMARKS.indexOf(landmark2); - const dist = Math.sqrt(((hand[index1].x - hand[index2].x) * image.width)**2 + ((hand[index1].y - hand[index2].y) * image.height)**2); + const dist = Math.sqrt( + ((hand[index1].x - hand[index2].x) * image.width) ** 2 + + ((hand[index1].y - hand[index2].y) * image.height) ** 2, + ); res.push(dist); } - + return res; } @@ -233,79 +269,98 @@ async function parseNormalLandmarkDist(image, landmark1, landmark2) { const data = await findHands(image); const coords = data.landmarks[0]; - - if(!coords || coords.length === 0) { + + if (!coords || coords.length === 0) { return []; } const index1 = LANDMARKS.indexOf(landmark1); const index2 = LANDMARKS.indexOf(landmark2); - const distance = Math.sqrt(((coords[index1].x - coords[index2].x))**2 + - ((coords[index1].y - coords[index2].y))**2 + - ((coords[index1].z - coords[index2].z))**2); - + const distance = Math.sqrt( + (coords[index1].x - coords[index2].x) ** 2 + + (coords[index1].y - coords[index2].y) ** 2 + + (coords[index1].z - coords[index2].z) ** 2, + ); + return distance; } -function updateConfig(option, newValue){ - switch(option) { - case 'Min Detect Confidence': +function updateConfig(option, newValue) { + switch (option) { + case "Min Detect Confidence": VisionHandler.config.options.minDetConf = newValue; - break; - case 'Min Presence Confidence': + break; + case "Min Presence Confidence": VisionHandler.config.options.minPresConf = newValue; break; - case 'Min Track Confidence': + case "Min Track Confidence": VisionHandler.config.options.minTracConf = newValue; break; - case 'Max Hands': + case "Max Hands": VisionHandler.config.options.maxHands = newValue; break; default: - throw Error('invalid Option'); + throw Error("invalid Option"); } } -async function updateAllHandleOptions(option, newValue){ - let min, max; +async function updateAllHandleOptions(option, newValue) { + let min, max; switch (option) { - case 'Max Hands': - min = 1; max = 4; + case "Max Hands": + min = 1; + max = 4; break; default: - min = 0; max = 1; + min = 0; + max = 1; break; } - if(!inRange(newValue, min, max)) { - throw new Error('Invalid Options Value'); + if (!inRange(newValue, min, max)) { + throw new Error("Invalid Options Value"); } - + updateConfig(option, newValue); - for(const handle of VISION_HANDLES){ + for (const handle of VISION_HANDLES) { handle.resetHandleOptions(); } } -function isValidLandmark(landmark){ +function isValidLandmark(landmark) { return LANDMARKS.indexOf(landmark) !== -1 ? true : false; } -function isValidDataOption(dataOption){ +function isValidDataOption(dataOption) { return DATAOPTIONS.indexOf(dataOption) !== -1 ? true : false; } -function isValidConfigOption(configOption){ +function isValidConfigOption(configOption) { return CONFIGOPTIONS.indexOf(configOption) !== -1 ? true : false; } -if(!document.getElementById('visionModule')){ - const script = document.createElement('script'); - script.id = 'visionModule'; - script.type = 'text/javascript'; +if (!document.getElementById("visionModule")) { + const script = document.createElement("script"); + script.id = "visionModule"; + script.type = "text/javascript"; script.src = DEVURL.VISIONMODULELOADERURL; script.async = false; - script.crossOrigin = 'anonymous'; + script.crossOrigin = "anonymous"; document.body.appendChild(script); } -export {VisionHandler, VISION_HANDLES, getVisionHandler, findHands, renderHands, parseLandmark, getCentralCoords, getCentralCoord, updateAllHandleOptions, isValidConfigOption, isValidDataOption, isValidLandmark, parseLandmarkDistance, parseNormalLandmarkDist} +export { + findHands, + getCentralCoord, + getCentralCoords, + getVisionHandler, + isValidConfigOption, + isValidDataOption, + isValidLandmark, + parseLandmark, + parseLandmarkDistance, + parseNormalLandmarkDist, + renderHands, + updateAllHandleOptions, + VISION_HANDLES, + VisionHandler, +}; diff --git a/extensions/HandGestures/index.js b/extensions/HandGestures/index.js index 0324ef6..f3f91f0 100644 --- a/extensions/HandGestures/index.js +++ b/extensions/HandGestures/index.js @@ -1,137 +1,208 @@ (async function () { - const localhost = window.location.search.includes('localhost'); - const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/'; + const localhost = window.location.search.includes("localhost"); + const root = localhost + ? "http://localhost:8000/" + : "https://extensions.netsblox.org/"; - const moduleURL = root + 'extensions/HandGestures/handLandmarkerModule.mjs'; + const moduleURL = root + "extensions/HandGestures/handLandmarkerModule.mjs"; const handModule = await import(moduleURL); - + function snapify(value) { if (Array.isArray(value)) { const res = []; for (const item of value) res.push(snapify(item)); return new List(res); - } else if (typeof(value) === 'object') { + } else if (typeof value === "object") { const res = []; for (const key in value) res.push(new List([key, snapify(value[key])])); return new List(res); } else return value; } - + class HandGestures extends Extension { constructor(ide) { - super('HandGestures'); + super("HandGestures"); this.ide = ide; } onOpenRole() {} - getMenu() { return {}; } + getMenu() { + return {}; + } - getCategories() { return []; } + getCategories() { + return []; + } getPalette() { const blocks = [ - '-', - new Extension.Palette.Block('handLandmarksFindHands'), - new Extension.Palette.Block('handLandmarksRender'), - new Extension.Palette.Block('handLandmarksFindLandmarks'), - new Extension.Palette.Block('handLandmarksFindLandmark'), - new Extension.Palette.Block('handLandmarksDistance'), - new Extension.Palette.Block('handLandmarksSetOptions'), - '-' + "-", + new Extension.Palette.Block("handLandmarksFindHands"), + new Extension.Palette.Block("handLandmarksRender"), + new Extension.Palette.Block("handLandmarksFindLandmarks"), + new Extension.Palette.Block("handLandmarksFindLandmark"), + new Extension.Palette.Block("handLandmarksDistance"), + new Extension.Palette.Block("handLandmarksSetOptions"), + "-", ]; return [ - new Extension.PaletteCategory('sensing', blocks, SpriteMorph), - new Extension.PaletteCategory('sensing', blocks, StageMorph), + new Extension.PaletteCategory("sensing", blocks, SpriteMorph), + new Extension.PaletteCategory("sensing", blocks, StageMorph), ]; } getBlocks() { function block(name, type, category, spec, defaults, action) { - return new Extension.Block(name, type, category, spec, defaults, action).for(SpriteMorph, StageMorph) + return new Extension.Block(name, type, category, spec, defaults, action) + .for(SpriteMorph, StageMorph); } return [ - block('handLandmarksFindHands', 'reporter', 'sensing', 'hands data from %s', [], function (img) { - return this.runAsyncFn(async () => { - img = img?.contents || img; - if (!img || typeof(img) !== 'object' || !img.width || !img.height) throw Error('Expected an image as input'); + block( + "handLandmarksFindHands", + "reporter", + "sensing", + "hands data from %s", + [], + function (img) { + return this.runAsyncFn(async () => { + img = img?.contents || img; + if ( + !img || typeof img !== "object" || !img.width || !img.height + ) throw Error("Expected an image as input"); - const res = await handModule.findHands(img); - - return snapify(res); - }, { args: [], timeout: 10000 }); - }), - - block('handLandmarksRender', 'reporter', 'sensing', 'render hands %s', [''], function (img) { - return this.runAsyncFn(async () => { - img = img?.contents || img; - if (!img || typeof(img) !== 'object' || !img.width || !img.height) {throw Error('Expected an image as input');} - - const res = await handModule.renderHands(img); + const res = await handModule.findHands(img); - return new Costume(res);}, { args: [], timeout: 10000 }); - }), + return snapify(res); + }, { args: [], timeout: 10000 }); + }, + ), - block('handLandmarksFindLandmarks', 'reporter', 'sensing', '%handLandmarkGet from %s', ['Hand Landmarks', ''], function (option, img) { + block("handLandmarksRender", "reporter", "sensing", "render hands %s", [ + "", + ], function (img) { return this.runAsyncFn(async () => { img = img?.contents || img; - if (!img || typeof(img) !== 'object' || !img.width || !img.height) throw Error('Expected an image as input'); - - option = option?.toString(); - if(!option) throw Error('Select a valid option') - - const res = await handModule.findHands(img); - - if(!res.landmarks) return snapify(res); - if(option === 'Hand Landmarks') return snapify(res.landmarks); - if(option === 'World Landmarks') return snapify(res.worldLandmarks); - if(option === 'Handedness') return snapify(res.handedness); - - throw new Error('Block Error') - }, { args: [], timeout: 10000 }); - }), + if (!img || typeof img !== "object" || !img.width || !img.height) { + throw Error("Expected an image as input"); + } - block('handLandmarksFindLandmark', 'reporter', 'sensing', '%handLandmarkGetOne of %handLandmarks from %s', ['Hand Landmarks', 'INDEX_tip'], function (option, landmark, img) { - return this.runAsyncFn(async () => { - landmark = landmark?.toString(); - option = option?.toString(); - if (!landmark || !handModule.isValidLandmark(landmark)) throw Error('invalid landmark'); - if (!option || !handModule.isValidDataOption(option)) throw Error('invalid option'); - img = img?.contents || img; - if (!img || typeof(img) !== 'object' || !img.width || !img.height) throw Error('Expected an image as input'); + const res = await handModule.renderHands(img); - const coords = await handModule.parseLandmark(img, option, landmark); - - return snapify(coords); + return new Costume(res); }, { args: [], timeout: 10000 }); }), + block( + "handLandmarksFindLandmarks", + "reporter", + "sensing", + "%handLandmarkGet from %s", + ["Hand Landmarks", ""], + function (option, img) { + return this.runAsyncFn(async () => { + img = img?.contents || img; + if ( + !img || typeof img !== "object" || !img.width || !img.height + ) throw Error("Expected an image as input"); - block('handLandmarksDistance', 'reporter', 'sensing', '%handLandmarkGetOne Distance of %handLandmarks to %handLandmarks from %s', ['Hand Landmarks', 'WRIST', 'THUMB_tip'], function (option, landmark1, landmark2, img) { - return this.runAsyncFn(async () => { - - landmark1 = landmark1?.toString(); - landmark2 = landmark2?.toString(); - if (!landmark1 || !landmark2) throw Error('landmark not specified'); + option = option?.toString(); + if (!option) throw Error("Select a valid option"); - img = img?.contents || img; - if (!img || typeof(img) !== 'object' || !img.width || !img.height) {throw Error('Expected an image as input');} - - const distance = await handModule.parseLandmarkDistance(img, landmark1, landmark2); - - return snapify(distance);}, { args: [], timeout: 10000 }); - }), - - block('handLandmarksSetOptions', 'command', 'sensing', 'set %handLandmarkOptions to %n', ['Max Hands', 2], function (option, newValue) { - return this.runAsyncFn(async () => { - if(!handModule.isValidConfigOption(option)){ - throw new Error('option not valid'); - } + const res = await handModule.findHands(img); + + if (!res.landmarks) return snapify(res); + if (option === "Hand Landmarks") return snapify(res.landmarks); + if (option === "World Landmarks") { + return snapify(res.worldLandmarks); + } + if (option === "Handedness") return snapify(res.handedness); + + throw new Error("Block Error"); + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "handLandmarksFindLandmark", + "reporter", + "sensing", + "%handLandmarkGetOne of %handLandmarks from %s", + ["Hand Landmarks", "INDEX_tip"], + function (option, landmark, img) { + return this.runAsyncFn(async () => { + landmark = landmark?.toString(); + option = option?.toString(); + if ( + !landmark || !handModule.isValidLandmark(landmark) + ) throw Error("invalid landmark"); + if ( + !option || !handModule.isValidDataOption(option) + ) throw Error("invalid option"); + img = img?.contents || img; + if ( + !img || typeof img !== "object" || !img.width || !img.height + ) throw Error("Expected an image as input"); + + const coords = await handModule.parseLandmark( + img, + option, + landmark, + ); + + return snapify(coords); + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "handLandmarksDistance", + "reporter", + "sensing", + "%handLandmarkGetOne Distance of %handLandmarks to %handLandmarks from %s", + ["Hand Landmarks", "WRIST", "THUMB_tip"], + function (option, landmark1, landmark2, img) { + return this.runAsyncFn(async () => { + landmark1 = landmark1?.toString(); + landmark2 = landmark2?.toString(); + if (!landmark1 || !landmark2) { + throw Error("landmark not specified"); + } + + img = img?.contents || img; + if ( + !img || typeof img !== "object" || !img.width || !img.height + ) throw Error("Expected an image as input"); + + const distance = await handModule.parseLandmarkDistance( + img, + landmark1, + landmark2, + ); + + return snapify(distance); + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "handLandmarksSetOptions", + "command", + "sensing", + "set %handLandmarkOptions to %n", + ["Max Hands", 2], + function (option, newValue) { + return this.runAsyncFn(async () => { + if (!handModule.isValidConfigOption(option)) { + throw new Error("option not valid"); + } + + await handModule.updateAllHandleOptions(option, newValue); - await handModule.updateAllHandleOptions(option, newValue); - - return snapify(newValue);}, { args: [], timeout: 10000 }); - }), + return snapify(newValue); + }, { args: [], timeout: 10000 }); + }, + ), ]; } @@ -149,46 +220,81 @@ return res; } return [ - new Extension.LabelPart('handLandmarks', () => new InputSlotMorph( - null, // text - false, // numeric - unionMaps([ - identityMap(['WRIST']), - {'THUMB': identityMap(['THUMB_cmc','THUMB_mcp','THUMB_ip','THUMB_tip'])}, - {'INDEX': identityMap(['INDEX_mcp','INDEX_pip','INDEX_dip','INDEX_tip'])}, - {'MIDDLE': identityMap(['MIDDLE_mcp','MIDDLE_pip','MIDDLE_dip','MIDDLE_tip'])}, - {'RING': identityMap([ 'RING_mcp','RING_pip','RING_dip','RING_tip'])}, - {'PINKY': identityMap(['PINKY_mcp','PINKY_pip','PINKY_dip','PINKY_tip'])} - ]), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('handLandmarkOptions', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(['Min Detect Confidence', - 'Min Presence Confidence', - 'Min Track Confidence', - 'Max Hands' ]), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('handLandmarkGet', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(['Hand Landmarks', - 'World Landmarks', - 'Handedness' - ]), - true, // readonly (no arbitrary text) - )), - new Extension.LabelPart('handLandmarkGetOne', () => new InputSlotMorph( - null, // text - false, // numeric - identityMap(['Hand Landmarks', - 'World Landmarks', - ]), - true, // readonly (no arbitrary text) - )), - + new Extension.LabelPart("handLandmarks", () => + new InputSlotMorph( + null, // text + false, // numeric + unionMaps([ + identityMap(["WRIST"]), + { + "THUMB": identityMap([ + "THUMB_cmc", + "THUMB_mcp", + "THUMB_ip", + "THUMB_tip", + ]), + }, + { + "INDEX": identityMap([ + "INDEX_mcp", + "INDEX_pip", + "INDEX_dip", + "INDEX_tip", + ]), + }, + { + "MIDDLE": identityMap([ + "MIDDLE_mcp", + "MIDDLE_pip", + "MIDDLE_dip", + "MIDDLE_tip", + ]), + }, + { + "RING": identityMap([ + "RING_mcp", + "RING_pip", + "RING_dip", + "RING_tip", + ]), + }, + { + "PINKY": identityMap([ + "PINKY_mcp", + "PINKY_pip", + "PINKY_dip", + "PINKY_tip", + ]), + }, + ]), + true, // readonly (no arbitrary text) + )), + new Extension.LabelPart("handLandmarkOptions", () => + new InputSlotMorph( + null, // text + false, // numeric + identityMap([ + "Min Detect Confidence", + "Min Presence Confidence", + "Min Track Confidence", + "Max Hands", + ]), + true, // readonly (no arbitrary text) + )), + new Extension.LabelPart("handLandmarkGet", () => + new InputSlotMorph( + null, // text + false, // numeric + identityMap(["Hand Landmarks", "World Landmarks", "Handedness"]), + true, // readonly (no arbitrary text) + )), + new Extension.LabelPart("handLandmarkGetOne", () => + new InputSlotMorph( + null, // text + false, // numeric + identityMap(["Hand Landmarks", "World Landmarks"]), + true, // readonly (no arbitrary text) + )), ]; } } diff --git a/extensions/PoseLandmarker/index.js b/extensions/PoseLandmarker/index.js index 441f3a5..e3cf089 100644 --- a/extensions/PoseLandmarker/index.js +++ b/extensions/PoseLandmarker/index.js @@ -1,16 +1,18 @@ (async function () { - const localhost = window.location.search.includes('localhost'); - const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/'; - + const localhost = window.location.search.includes("localhost"); + const root = localhost + ? "http://localhost:8000/" + : "https://extensions.netsblox.org/"; + const DEVURL = { - MODELPATHURL: 'https://samankittani.github.io/mediapipe_models/models/pose_landmarker_full.task', - VISIONMODULELOADERURL: root + 'utils/visionModuleLoader.js' - } + MODELPATHURL: + "https://samankittani.github.io/mediapipe_models/models/pose_landmarker_full.task", + VISIONMODULELOADERURL: root + "utils/visionModuleLoader.js", + }; class PoseHandler { constructor() { - - this.poseLandmarker = null; + this.poseLandmarker = null; this.expiry = 0; this.resolve = null; @@ -26,46 +28,52 @@ minPoseDetConf: .5, minPosePresConf: .5, minTracConf: .5, - segMask: false - } - } + segMask: false, + }, + }; async generateTask() { - if(!Vision) - throw Error('Vision Module is loading...'); - - if(this.poseLandmarker === 'loading...') - throw Error('PoseLandmarker is currently loading'); - - this.poseLandmarker = 'loading...'; - - this.poseLandmarker = await Vision.Module.PoseLandmarker.createFromOptions(Vision.Task, { - baseOptions: { - modelAssetPath: DEVURL.MODELPATHURL, - delegate: 'GPU' - }, - numPoses: PoseHandler.config.options.numPoses, - runningMode: 'VIDEO', - minPoseDetectionConfidence: PoseHandler.config.options.minPoseDetConf, - minPosePresenceConfidence: PoseHandler.config.options.minPosePresConf, - minTrackingConfidence: PoseHandler.config.options.minTracConf - }).catch((err) => { - throw new Error(`Failed to generate vision task: Error: ${err}`); - }) + if (!Vision) { + throw Error("Vision Module is loading..."); + } + + if (this.poseLandmarker === "loading...") { + throw Error("PoseLandmarker is currently loading"); + } + + this.poseLandmarker = "loading..."; + + this.poseLandmarker = await Vision.Module.PoseLandmarker + .createFromOptions(Vision.Task, { + baseOptions: { + modelAssetPath: DEVURL.MODELPATHURL, + delegate: "GPU", + }, + numPoses: PoseHandler.config.options.numPoses, + runningMode: "VIDEO", + minPoseDetectionConfidence: PoseHandler.config.options.minPoseDetConf, + minPosePresenceConfidence: PoseHandler.config.options.minPosePresConf, + minTrackingConfidence: PoseHandler.config.options.minTracConf, + }).catch((err) => { + throw new Error(`Failed to generate vision task: Error: ${err}`); + }); } - async infer(image){ - if(this.poseLandmarker === null){ - throw new Error('poseLandmarker is not initialized'); + async infer(image) { + if (this.poseLandmarker === null) { + await this.generateTask(); } - if(this.poseLandmarker === 'loading...'){ - throw new Error ('poseLandmarker is loading...'); + if (this.poseLandmarker === "loading...") { + throw new Error("poseLandmarker is loading..."); } - if(this.resolve !== null) throw Error('PoseHandler is currently in use'); - this.resolve = 'loading...'; + if (this.resolve !== null) throw Error("PoseHandler is currently in use"); + this.resolve = "loading..."; this.frameTime = performance.now(); - if(PoseHandler.config.data !== null && ((this.frameTime - PoseHandler.config.updateTime) < 10)){ + if ( + PoseHandler.config.data !== null && + ((this.frameTime - PoseHandler.config.updateTime) < 10) + ) { this.resolve = null; return PoseHandler.config.data; } @@ -73,11 +81,14 @@ this.expiry = +new Date() + 10000; - return new Promise(resolve => { - PoseHandler.config.data = this.poseLandmarker.detectForVideo(image, this.frameTime); + return new Promise((resolve) => { + PoseHandler.config.data = this.poseLandmarker.detectForVideo( + image, + this.frameTime, + ); this.resolve = null; resolve(PoseHandler.config.data); - }) + }); } isIdle() { @@ -88,13 +99,14 @@ } return false; } - } + } const POSE_HANDLES = []; function getPoseHandler() { for (const handle of POSE_HANDLES) { - if (handle.isIdle()) + if (handle.isIdle()) { return handle; + } } const handle = new PoseHandler(); POSE_HANDLES.push(handle); @@ -109,11 +121,14 @@ async function renderPose(image) { const data = await findPose(image); - const context = image.getContext('2d'); + const context = image.getContext("2d"); const drawer = new Vision.Module.DrawingUtils(context); for (const landmarks of data.landmarks) { - drawer.drawConnectors(landmarks, Vision.Module.PoseLandmarker.POSE_CONNECTIONS); + drawer.drawConnectors( + landmarks, + Vision.Module.PoseLandmarker.POSE_CONNECTIONS, + ); drawer.drawLandmarks(landmarks); } return image; @@ -124,61 +139,84 @@ const res = []; for (const item of value) res.push(snapify(item)); return new List(res); - } else if (typeof(value) === 'object') { + } else if (typeof value === "object") { const res = []; for (const key in value) res.push(new List([key, snapify(value[key])])); return new List(res); } else return value; } - + class PoseLandmarker extends Extension { constructor(ide) { - super('PoseLandmarker'); + super("PoseLandmarker"); this.ide = ide; } onOpenRole() {} - getMenu() { return {}; } + getMenu() { + return {}; + } - getCategories() { return []; } + getCategories() { + return []; + } getPalette() { const blocks = [ - new Extension.Palette.Block('poseLandmarksFindPose'), - new Extension.Palette.Block('poseLandmarksRender'), + new Extension.Palette.Block("poseLandmarksFindPose"), + new Extension.Palette.Block("poseLandmarksRender"), ]; return [ - new Extension.PaletteCategory('sensing', blocks, SpriteMorph), - new Extension.PaletteCategory('sensing', blocks, StageMorph), + new Extension.PaletteCategory("sensing", blocks, SpriteMorph), + new Extension.PaletteCategory("sensing", blocks, StageMorph), ]; } getBlocks() { function block(name, type, category, spec, defaults, action) { - return new Extension.Block(name, type, category, spec, defaults, action).for(SpriteMorph, StageMorph) + return new Extension.Block(name, type, category, spec, defaults, action) + .for(SpriteMorph, StageMorph); } return [ - block('poseLandmarksFindPose', 'reporter', 'sensing', 'pose data from %s', [], function (img) { - return this.runAsyncFn(async () => { - img = img?.contents || img; - if (!img || typeof(img) !== 'object' || !img.width || !img.height) throw Error('Expected an image as input'); - - const res = await findPose(img); - return snapify(res); - }, { args: [], timeout: 10000 }); - }), - - block('poseLandmarksRender', 'reporter', 'sensing', 'render pose from %s', [''], function (img) { - return this.runAsyncFn(async () => { - img = img?.contents || img; - - if (!img || typeof(img) !== 'object' || !img.width || !img.height) {throw Error('Expected an image as input');} - - const res = await renderPose(img); - return new Costume(res); - }, { args: [], timeout: 10000 }); - }), + block( + "poseLandmarksFindPose", + "reporter", + "sensing", + "pose data from %s", + [], + function (img) { + return this.runAsyncFn(async () => { + img = img?.contents || img; + if ( + !img || typeof img !== "object" || !img.width || !img.height + ) throw Error("Expected an image as input"); + + const res = await findPose(img); + return snapify(res); + }, { args: [], timeout: 10000 }); + }, + ), + + block( + "poseLandmarksRender", + "reporter", + "sensing", + "render pose from %s", + [""], + function (img) { + return this.runAsyncFn(async () => { + img = img?.contents || img; + + if ( + !img || typeof img !== "object" || !img.width || !img.height + ) throw Error("Expected an image as input"); + + const res = await renderPose(img); + return new Costume(res); + }, { args: [], timeout: 10000 }); + }, + ), ]; } @@ -188,19 +226,18 @@ for (const x of s) res[x] = x; return res; } - return [ - ]; + return []; } } const visionSource = DEVURL.VISIONMODULELOADERURL; - if(!document.getElementById('visionModule')){ - const script = document.createElement('script'); - script.id = 'visionModule'; - script.type = 'text/javascript'; + if (!document.getElementById("visionModule")) { + const script = document.createElement("script"); + script.id = "visionModule"; + script.type = "text/javascript"; script.src = visionSource; script.async = false; - script.crossOrigin = 'anonymous'; + script.crossOrigin = "anonymous"; document.body.appendChild(script); }