Skip to content

Commit

Permalink
performanceFix (#58)
Browse files Browse the repository at this point in the history 
* added two new mini dictionaries and the ability to set a dictionary

* removed console log
  • Loading branch information
@samankittani
samankittani authored Jan 24, 2024
1 parent ae1dbb2 commit 48c8681
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 25 deletions.
59 changes: 47 additions & 12 deletions extensions/AugmentedReality/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
(async function () {

let videoMirrored = false;

const dictionaries = ['APRILTAG_16h5', 'APRILTAG_16h5_mini','APRILTAG_16h5_duo'];

const localhost = window.location.search.includes('localhost');
const root = localhost? 'http://localhost:8000/' : 'https://extensions.netsblox.org/';
Expand Down Expand Up @@ -46,12 +48,16 @@
getPalette() {
const blocks = [
new Extension.Palette.Block('ARCodeTracker'),
new Extension.Palette.Block('ARCodeRender'),
'-',
new Extension.Palette.Block('ARCodeFlag'),
new Extension.Palette.Block('ARCodeVisibleArray'),
new Extension.Palette.Block('ARCodeRender'),
'-',
new Extension.Palette.Block('ARCodeSetDictionary'),
new Extension.Palette.Block('ARCodeDictionary').withWatcherToggle(),
'-',
new Extension.Palette.Block('ARCodeFlipVideo'),
new Extension.Palette.Block('flipped').withWatcherToggle(),
new Extension.Palette.Block('ARCodeFlipped').withWatcherToggle(),
'-'
];
return [
Expand All @@ -78,7 +84,23 @@
return snapify(res);

}, { args: [], timeout: 10000 });
}),
}),


block('ARCodeRender', 'reporter', 'sensing', 'render %model on %s', ['box'], function (model, image) {
return this.runAsyncFn(async () => {

image = image?.contents || image;
if (!image || typeof(image) !== 'object' || !image.width || !image.height){
throw TypeError('Expected an image as input');
}
const res = await renderModule.renderScene(image, model);

return new Costume(res);

}, { args: [], timeout: 10000 });
}),


block('ARCodeFlag', 'predicate', 'sensing', 'AR code %n visible in %s ?', [], function (value, image) {
return this.runAsyncFn(async () => {
Expand Down Expand Up @@ -113,9 +135,9 @@
}, { args: [], timeout: 10000 });
}),


block('ARCodeVisibleArray', 'reporter', 'sensing', 'All AR codes visible in %s', [], function (image) {
return this.runAsyncFn(async () => {
const begin = performance.now();

image = image?.contents || image;
if (!image || typeof(image) !== 'object' || !image.width || !image.height){
Expand All @@ -124,26 +146,31 @@

const visible = await tagModule.getVisibleTags(image);

console.log(performance.now() - begin);
return snapify(visible);

}, { args: [], timeout: 10000 });
}),

block('ARCodeRender', 'reporter', 'sensing', 'render %model on %s', ['box'], function (model, image) {

block('ARCodeSetDictionary', 'command', 'sensing', 'set dictionary to %dictionaries', ['APRILTAG_16h5'], function (dict) {
return this.runAsyncFn(async () => {

image = image?.contents || image;
if (!image || typeof(image) !== 'object' || !image.width || !image.height){
throw TypeError('Expected an image as input');
console.log(dict, dictionaries.indexOf(dict) );
if(dictionaries.indexOf(dict) === -1){
return new Error(dict, 'is not a valid dictionary.');
}
const res = await renderModule.renderScene(image, model);

return new Costume(res);
tagModule.setDictionary(dict)

}, { args: [], timeout: 10000 });
}),


block('ARCodeDictionary', 'reporter', 'sensing', 'dictionary', [], function () {
return tagModule.getDictionary();
}),


block('ARCodeFlipVideo', 'command', 'sensing', 'flip video', [], function () {
return this.runAsyncFn(async () => {

Expand All @@ -153,14 +180,16 @@
}, { args: [], timeout: 10000 });
}),

block('flipped', 'reporter', 'sensing', 'flipped?', [], function () {

block('ARCodeFlipped', 'reporter', 'sensing', 'flipped?', [], function () {

videoMirrored = world.children[0].stage.mirrorVideo;
return snapify(videoMirrored);
})
];
}


getLabelParts() {
function identityMap(s) {
const res = {};
Expand All @@ -182,6 +211,12 @@
identityMap(['box', 'drum', 'piano']),
true
)),
new Extension.LabelPart('dictionaries', () => new InputSlotMorph(
null, // text
false, // numeric
identityMap(dictionaries),
true
)),
];
}
}
Expand Down
57 changes: 44 additions & 13 deletions extensions/AugmentedReality/js/tagHandler.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@ const positVersion = 1;

class aprilTagHandler {
constructor() {
this.detector = new AR.Detector({dictionaryName: 'APRILTAG_16h5'});
this.detector = new AR.Detector({dictionaryName: aprilTagHandler.config.dictionary, maxHammingDistance: 3});
this.state = 'Idle';
this.expiry = 0;
}

static config = {
dictionary: 'APRILTAG_16h5'
}

async detectAR(imageData){
this.state = 'Detecting...';
this.expiry = +new Date() + 10000;
Expand All @@ -28,23 +33,32 @@ const APRILTAGHANDLERS = [];

function getAprilHandler(){
for(const handler of APRILTAGHANDLERS){
if(handler.isIdle())
if(handler.isIdle()){
return handler;
}
}
const newHandler = new aprilTagHandler();
APRILTAGHANDLERS.push(newHandler);
return newHandler;
}

function resetAllDetectors(dict){

for(const handler of APRILTAGHANDLERS){
(async (handler, dict) => {
console.log(handler, dict);
handler.detector = new AR.Detector({dictionaryName: dict});
console.log(handler, dict);
})(handler, dict);
}
}

async function getCoordinates(image){
const handler = getAprilHandler();
const context = image.getContext('2d');
const imageData = context.getImageData(0,0,image.width, image.height);

const markers = await handler.detectAR(imageData);
console.log('markers:');
console.log(markers);

return markers;
}

Expand All @@ -53,8 +67,8 @@ function transformCoordinates(markers, image){
return markers;
}

for(let marker of markers){
for(let corner of marker.corners){
for(const marker of markers){
for(const corner of marker.corners){
corner.x = corner.x - (image.width/2);
corner.y = 1 - (corner.y - (image.height/2));
}
Expand All @@ -64,7 +78,7 @@ function transformCoordinates(markers, image){

async function isTagVisible(image, value) {
const markers = await getCoordinates(image);
for(let marker of markers){
for(const marker of markers){
if(value.indexOf(marker.id) !== -1){
return true;
}
Expand All @@ -73,14 +87,27 @@ async function isTagVisible(image, value) {
}

async function getVisibleTags(image) {
const res = (new Array(30)).fill(false);
const handler = getAprilHandler();
const res = (new Array(handler.detector.dictionary.codeList.length)).fill(false);
const markers = await getCoordinates(image);
for(let marker of markers){
for(const marker of markers){
res[marker.id] = true;
}
return res;
}

function getDictionary(){
return aprilTagHandler.config.dictionary;
}

function setDictionary(dict){
if(dict === aprilTagHandler.config.dictionary){
return;
}
aprilTagHandler.config.dictionary = dict;
resetAllDetectors(dict);
}

const DEVURL = {
SVDURL:
'https://samankittani.github.io/js-aruco2/src/svd.js',
Expand All @@ -90,8 +117,12 @@ const DEVURL = {
'https://samankittani.github.io/js-aruco2/src/cv.js',
ARUCOURL:
'https://samankittani.github.io/js-aruco2/src/aruco.js',
APRILTAGURL:
'https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5.js'
TAGDICURL01:
'https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5.js',
TAGDICURL02:
'https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5_duo.js',
TAGDICURL03:
'https://samankittani.github.io/js-aruco2/src/dictionaries/apriltag_16h5_mini.js'
}
const sources = Object.values(DEVURL);

Expand All @@ -107,4 +138,4 @@ const DEVURL = {
}
}

export {getCoordinates, transformCoordinates, isTagVisible, getVisibleTags}
export {getCoordinates, transformCoordinates, isTagVisible, getVisibleTags, getDictionary, setDictionary}

0 comments on commit 48c8681

Please sign in to comment.