>4,g=15&e[t+1],w=e[t+2];d.componentsOrder.push(m),d.components[m]={h:y,v:g,quantizationIdx:w},t+=3}n(d),this.frames.push(d);break;case 65476:for(var b=r(),k=2;k>4==0?this.huffmanTablesDC[15&x]=en(A,P):this.huffmanTablesAC[15&x]=en(A,P)}break;case 65501:r(),this.resetInterval=r();break;case 65498:r();for(var T=e[t++],U=[],O=this.frames[0],D=0;D 3)throw"Invalid block encoding ("+b.encoding+")";if(2!==b.encoding){if(0!==k&&2!==k){if(k>>=6,b.offsetType=k,2===k)b.offset=a.getInt8(1),g++;else if(1===k)b.offset=a.getInt16(1,!0),g+=2;else{if(0!==k)throw"Invalid block offset type";b.offset=a.getFloat32(1,!0),g+=4}if(1===b.encoding)if(k=a.getUint8(g),g++,b.bitsPerPixel=63&k,k>>=6,b.numValidPixelsType=k,2===k)b.numValidPixels=a.getUint8(g),g++;else if(1===k)b.numValidPixels=a.getUint16(g,!0),g+=2;else{if(0!==k)throw"Invalid valid pixel count type";b.numValidPixels=a.getUint32(g,!0),g+=4}}var x;if(t+=g,3!==b.encoding)if(0===b.encoding){var A=(n.pixels.numBytes-1)/4;if(A!==Math.floor(A))throw"uncompressed block has invalid length";x=new ArrayBuffer(4*A),new Uint8Array(x).set(new Uint8Array(e,t,4*A));var _=new Float32Array(x);b.rawData=_,t+=4*A}else if(1===b.encoding){var I=Math.ceil(b.numValidPixels*b.bitsPerPixel/8),P=Math.ceil(I/4);x=new ArrayBuffer(4*P),new Uint8Array(x).set(new Uint8Array(e,t,I)),b.stuffedData=new Uint32Array(x),t+=I}}else t++}return n.eofOffset=t,n},o=function(e,t,r,n,i,a,o){var s,u,f,c=(1< 1&&(R=new r(E.buffer,y*U,y),T=0),t.headerInfo.numValidPixel===v*m)for(h=0,c=0;c >4,w=15&e[t+1],g=e[t+2];d.componentsOrder.push(m),d.components[m]={h:y,v:w,quantizationIdx:g},t+=3}n(d),this.frames.push(d);break;case 65476:for(var b=r(),k=2;k>4==0?this.huffmanTablesDC[15&x]=E(_,I):this.huffmanTablesAC[15&x]=E(_,I)}break;case 65501:r(),this.resetInterval=r();break;case 65498:r();for(var S=e[t++],T=[],O=this.frames[0],M=0;M >>=v,u-=v,0===r.have){e.msg="invalid bit length repeat",r.mode=30;break}x=r.lens[r.have-1],d=3+(3&l),l>>>=2,u-=2}else if(17===w){for(P=v+3;u >>=v)),l>>>=3,u-=3}else{for(P=v+7;u >>=v)),l>>>=7,u-=7}if(r.have+d>r.nlen+r.ndist){e.msg="invalid bit length repeat",r.mode=30;break}for(;d--;)r.lens[r.have++]=x}}if(30===r.mode)break;if(0===r.lens[256]){e.msg="invalid code -- missing end-of-block",r.mode=30;break}if(r.lenbits=9,A={bits:r.lenbits},_=ie(1,r.lens,0,r.nlen,r.lencode,0,r.work,A),r.lenbits=A.bits,_){e.msg="invalid literal/lengths set",r.mode=30;break}if(r.distbits=6,r.distcode=r.distdyn,A={bits:r.distbits},_=ie(2,r.lens,r.nlen,r.ndist,r.distcode,0,r.work,A),r.distbits=A.bits,_){e.msg="invalid distances set",r.mode=30;break}if(r.mode=20,t===se)break e;case 20:r.mode=21;case 21:if(s>=6&&f>=258){e.next_out=o,e.avail_out=f,e.next_in=a,e.avail_in=s,r.hold=l,r.bits=u,$(e,h),o=e.next_out,i=e.output,f=e.avail_out,a=e.next_in,n=e.input,s=e.avail_in,l=r.hold,u=r.bits,12===r.mode&&(r.back=-1);break}for(r.back=0;y=(I=r.lencode[l&(1< >>=r.extra,u-=r.extra,r.back+=r.extra}r.was=r.length,r.mode=23;case 23:for(;y=(I=r.distcode[l&(1< >>=r.extra,u-=r.extra,r.back+=r.extra}if(r.offset>r.dmax){e.msg="invalid distance too far back",r.mode=30;break}r.mode=25;case 25:if(0===f)break e;if(d=h-f,r.offset>d){if((d=r.offset-d)>r.whave&&r.sane){e.msg="invalid distance too far back",r.mode=30;break}d>r.wnext?(d-=r.wnext,p=r.wsize-d):p=r.wnext-d,d>r.length&&(d=r.length),m=r.window}else m=i,p=o-r.offset,d=r.length;d>f&&(d=f),f-=d,r.length-=d;do{i[o++]=m[p++]}while(--d);0===r.length&&(r.mode=21);break;case 26:if(0===f)break e;i[o++]=r.length,f--,r.mode=21;break;case 27:if(r.wrap){for(;u<32;){if(0===s)break e;s--,l|=n[a++]<=0&&t.windowBits<16&&(t.windowBits=-t.windowBits,0===t.windowBits&&(t.windowBits=-15)),!(t.windowBits>=0&&t.windowBits<16)||e&&e.windowBits||(t.windowBits+=32),t.windowBits>15&&t.windowBits<48&&0==(15&t.windowBits)&&(t.windowBits|=15),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new Q,this.strm.avail_out=0;var r=Ue.inflateInit2(this.strm,t.windowBits);if(r!==Be)throw new Error(z[r]);if(this.header=new Se,Ue.inflateGetHeader(this.strm,this.header),t.dictionary&&("string"==typeof t.dictionary?t.dictionary=W(t.dictionary):"[object ArrayBuffer]"===Te.call(t.dictionary)&&(t.dictionary=new Uint8Array(t.dictionary)),t.raw&&(r=Ue.inflateSetDictionary(this.strm,t.dictionary))!==Be))throw new Error(z[r])}function Ge(e,t){var r=new Ve(t);if(r.push(e),r.err)throw r.msg||z[r.err];return r.result}Ve.prototype.push=function(e,t){var r,n,i,a=this.strm,o=this.options.chunkSize,s=this.options.dictionary;if(this.ended)return!1;for(n=t===~~t?t:!0===t?Ee:De,"[object ArrayBuffer]"===Te.call(e)?a.input=new Uint8Array(e):a.input=e,a.next_in=0,a.avail_in=a.input.length;;){for(0===a.avail_out&&(a.output=new Uint8Array(o),a.next_out=0,a.avail_out=o),(r=Ue.inflate(a,n))===Me&&s&&((r=Ue.inflateSetDictionary(a,s))===Be?r=Ue.inflate(a,n):r===Le&&(r=Me));a.avail_in>0&&r===Oe&&a.state.wrap>0&&0!==e[a.next_in];)Ue.inflateReset(a),r=Ue.inflate(a,n);switch(r){case Ce:case Le:case Me:case Re:return this.onEnd(r),this.ended=!0,!1}if(i=a.avail_out,a.next_out&&(0===a.avail_out||r===Oe))if("string"===this.options.to){var f=J(a.output,a.next_out),l=a.next_out-f,u=q(a.output,f);a.next_out=l,a.avail_out=o-l,l&&a.output.set(a.output.subarray(f,f+l),0),this.onData(u)}else this.onData(a.output.length===a.next_out?a.output:a.output.subarray(0,a.next_out));if(r!==Be||0!==i){if(r===Oe)return r=Ue.inflateEnd(this.strm),this.onEnd(r),this.ended=!0,!0;if(0===a.avail_in)break}}return!0},Ve.prototype.onData=function(e){this.chunks.push(e)},Ve.prototype.onEnd=function(e){e===Be&&("string"===this.options.to?this.result=this.chunks.join(""):this.result=K(this.chunks)),this.chunks=[],this.err=e,this.msg=this.strm.msg};var Fe={Inflate:Ve,inflate:Ge,inflateRaw:function(e,t){return(t=t||{}).raw=!0,Ge(e,t)},ungzip:Ge,constants:j}.inflate;function ze(e){var t=function(){if("undefined"==typeof Reflect||!Reflect.construct)return!1;if(Reflect.construct.sham)return!1;if("function"==typeof Proxy)return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],(function(){}))),!0}catch(e){return!1}}();return function(){var r,n=v(e);if(t){var i=v(this).constructor;r=Reflect.construct(n,arguments,i)}else r=n.apply(this,arguments);return m(this,r)}}var je=function(e){p(r,b);var t=ze(r);function r(){return u(this,r),t.apply(this,arguments)}return h(r,[{key:"decodeBlock",value:function(e){return Fe(new Uint8Array(e)).buffer}}]),r}(),Ne=Object.freeze({__proto__:null,default:je});function Ze(e){var t=function(){if("undefined"==typeof Reflect||!Reflect.construct)return!1;if(Reflect.construct.sham)return!1;if("function"==typeof Proxy)return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],(function(){}))),!0}catch(e){return!1}}();return function(){var r,n=v(e);if(t){var i=v(this).constructor;r=Reflect.construct(n,arguments,i)}else r=n.apply(this,arguments);return m(this,r)}}var Ke,He=function(e){p(r,b);var t=Ze(r);function r(){return u(this,r),t.apply(this,arguments)}return h(r,[{key:"decodeBlock",value:function(e){for(var t=new DataView(e),r=[],n=0;n 3)throw"Invalid block encoding ("+b.encoding+")";if(2!==b.encoding){if(0!==k&&2!==k){if(k>>=6,b.offsetType=k,2===k)b.offset=a.getInt8(1),w++;else if(1===k)b.offset=a.getInt16(1,!0),w+=2;else{if(0!==k)throw"Invalid block offset type";b.offset=a.getFloat32(1,!0),w+=4}if(1===b.encoding)if(k=a.getUint8(w),w++,b.bitsPerPixel=63&k,k>>=6,b.numValidPixelsType=k,2===k)b.numValidPixels=a.getUint8(w),w++;else if(1===k)b.numValidPixels=a.getUint16(w,!0),w+=2;else{if(0!==k)throw"Invalid valid pixel count type";b.numValidPixels=a.getUint32(w,!0),w+=4}}var x;if(t+=w,3!==b.encoding)if(0===b.encoding){var _=(n.pixels.numBytes-1)/4;if(_!==Math.floor(_))throw"uncompressed block has invalid length";x=new ArrayBuffer(4*_),new Uint8Array(x).set(new Uint8Array(e,t,4*_));var A=new Float32Array(x);b.rawData=A,t+=4*_}else if(1===b.encoding){var P=Math.ceil(b.numValidPixels*b.bitsPerPixel/8),I=Math.ceil(P/4);x=new ArrayBuffer(4*I),new Uint8Array(x).set(new Uint8Array(e,t,P)),b.stuffedData=new Uint32Array(x),t+=P}}else t++}return n.eofOffset=t,n},a=function(e,t,r,n,i,a,o){var s,f,l,u=(1< 1&&(O=new r(B.buffer,y*T,y),S=0),t.headerInfo.numValidPixel===v*m)for(c=0,l=0;l >>64-_-P),i=b,d=0;d >>P-d-1&1?i.right:i.left).left&&!i.right){a=i.val,_=_+d+1;break}_>=32&&(_-=32,D=k[++x]),o=a-I,E?(o+=u>0?S:l>0?O[c-v]:S,o&=255,O[c]=o,S=o):O[c]=o}else for(c=0,l=0;l >>64-_-P),i=b,d=0;d >>P-d-1&1?i.right:i.left).left&&!i.right){a=i.val,_=_+d+1;break}_>=32&&(_-=32,D=k[++x]),o=a-I,E?(u>0&&U[c-1]?o+=S:l>0&&U[c-v]?o+=O[c-v]:o+=S,o&=255,O[c]=o,S=o):O[c]=o}}else for(c=0,l=0;l >>64-_-P),i=b,d=0;d >>P-d-1&1?i.right:i.left).left&&!i.right){a=i.val,_=_+d+1;break}_>=32&&(_-=32,D=k[++x]),o=a-I,O[c]=o}t.ptr=t.ptr+4*(x+1)+(_>0?4:0),t.pixels.resultPixels=B,p>1&&!n&&(t.pixels.resultPixels=h.swapDimensionOrder(B,y,p,r))},decodeBits:function(e,t,r,n,i){var a=t.headerInfo,h=a.fileVersion,d=0,p=e.byteLength-t.ptr>=5?5:e.byteLength-t.ptr,m=new DataView(e,t.ptr,p),v=m.getUint8(0);d++;var y=v>>6,w=0===y?4:3-y,g=(32&v)>0,b=31&v,k=0;if(1===w)k=m.getUint8(d),d++;else if(2===w)k=m.getUint16(d,!0),d+=2;else{if(4!==w)throw"Invalid valid pixel count type";k=m.getUint32(d,!0),d+=4}var x,_,A,P,I,U,S,T,D,E=2*a.maxZError,B=a.numDims>1?a.maxValues[i]:a.zMax;if(g){for(t.counter.lut++,T=m.getUint8(d),d++,P=Math.ceil((T-1)*b/8),I=Math.ceil(P/4),_=new ArrayBuffer(4*I),A=new Uint8Array(_),t.ptr+=d,A.set(new Uint8Array(e,t.ptr,P)),S=new Uint32Array(_),t.ptr+=P,D=0;T-1>>>D;)D++;P=Math.ceil(k*D/8),I=Math.ceil(P/4),_=new ArrayBuffer(4*I),(A=new Uint8Array(_)).set(new Uint8Array(e,t.ptr,P)),x=new Uint32Array(_),t.ptr+=P,U=h>=3?l(S,b,T-1,n,E,B):s(S,b,T-1,n,E,B),h>=3?f(x,r,D,k,U):o(x,r,D,k,U)}else t.counter.bitstuffer++,D=b,t.ptr+=d,D>0&&(P=Math.ceil(k*D/8),I=Math.ceil(P/4),_=new ArrayBuffer(4*I),(A=new Uint8Array(_)).set(new Uint8Array(e,t.ptr,P)),x=new Uint32Array(_),t.ptr+=P,h>=3?null==n?c(x,r,D,k):f(x,r,D,k,!1,n,E,B):null==n?u(x,r,D,k):o(x,r,D,k,!1,n,E,B))},readTiles:function(e,t,r,n){var i=t.headerInfo,a=i.width,o=i.height,s=a*o,f=i.microBlockSize,l=i.imageType,u=h.getDataTypeSize(l),c=Math.ceil(a/f),d=Math.ceil(o/f);t.pixels.numBlocksY=d,t.pixels.numBlocksX=c,t.pixels.ptr=0;var p,m,v,y,w,g,b,k,x,_,A=0,P=0,I=0,U=0,S=0,T=0,D=0,E=0,B=0,O=0,M=0,C=0,L=0,R=0,V=0,G=new r(f*f),F=o%f||f,z=a%f||f,j=i.numDims,N=t.pixels.resultMask,Z=t.pixels.resultPixels,K=i.fileVersion>=5?14:15,H=i.zMax;for(I=0;I>9&1,r.head.done=!0),e.adler=r.check=0,r.mode=jn;break;case 10:for(;c<32;){if(0===s)break e;s--,f+=n[a++]<=359?359:i;i-=o;do{t+=e[a++]<<8,r+=t+=e[a++]}while(--o);t=(65535&t)+(t>>>16),r=(65535&r)+(r>>>16)}return 1&n&&(r+=t+=e[a]<<8),((r=(65535&r)+(r>>>16))<<16|(t=(65535&t)+(t>>>16)))>>>0},readHeaderInfo:function(e,t){var r=t.ptr,n=new Uint8Array(e,r,6),i={};if(i.fileIdentifierString=String.fromCharCode.apply(null,n),0!==i.fileIdentifierString.lastIndexOf("Lerc2",0))throw"Unexpected file identifier string (expect Lerc2 ): "+i.fileIdentifierString;r+=6;var a,o=new DataView(e,r,8),s=o.getInt32(0,!0);if(i.fileVersion=s,r+=4,s>=3&&(i.checksum=o.getUint32(4,!0),r+=4),o=new DataView(e,r,12),i.height=o.getUint32(0,!0),i.width=o.getUint32(4,!0),r+=8,s>=4?(i.numDims=o.getUint32(8,!0),r+=4):i.numDims=1,o=new DataView(e,r,40),i.numValidPixel=o.getUint32(0,!0),i.microBlockSize=o.getInt32(4,!0),i.blobSize=o.getInt32(8,!0),i.imageType=o.getInt32(12,!0),i.maxZError=o.getFloat64(16,!0),i.zMin=o.getFloat64(24,!0),i.zMax=o.getFloat64(32,!0),r+=40,t.headerInfo=i,t.ptr=r,s>=3&&(a=s>=4?52:48,this.computeChecksumFletcher32(new Uint8Array(e,r-a,i.blobSize-14))!==i.checksum))throw"Checksum failed.";return!0},checkMinMaxRanges:function(e,t){var r=t.headerInfo,n=this.getDataTypeArray(r.imageType),i=r.numDims*this.getDataTypeSize(r.imageType),a=this.readSubArray(e,t.ptr,n,i),o=this.readSubArray(e,t.ptr+i,n,i);t.ptr+=2*i;var s,u=!0;for(s=0;s>4],C.huffmanTableAC=this.huffmanTablesAC[15&L],T.push(C)}var R=e[t++],V=e[t++],G=e[t++],F=B(e,t,O,T,this.resetInterval,R,V,G>>4,15&G);t+=F;break;case 65535:255!==e[t]&&t--;break;default:if(255===e[t-3]&&e[t-2]>=192&&e[t-2]<=254){t-=3;break}throw new Error("unknown JPEG marker ".concat(o.toString(16)))}o=r()}}},{key:"getResult",value:function(){var e=this.frames;if(0===this.frames.length)throw new Error("no frames were decoded");this.frames.length>1&&console.warn("more than one frame is not supported");for(var t=0;t>9&1,r.head.done=!0),e.adler=r.check=0,r.mode=12;break;case 10:for(;u<32;){if(0===s)break e;s--,l+=n[a++]<>>=7&u,u-=7&u,r.mode=27;break}for(;u<3;){if(0===s)break e;s--,l+=n[a++]<>>=1)){case 0:r.mode=14;break;case 1:if(Pe(r),r.mode=20,t===se){l>>>=2,u-=2;break e}break;case 2:r.mode=17;break;case 3:e.msg="invalid block type",r.mode=30}l>>>=2,u-=2;break;case 14:for(l>>>=7&u,u-=7&u;u<32;){if(0===s)break e;s--,l+=n[a++]<>>16^65535)){e.msg="invalid stored block lengths",r.mode=30;break}if(r.length=65535&l,l=0,u=0,r.mode=15,t===se)break e;case 15:r.mode=16;case 16:if(d=r.length){if(d>s&&(d=s),d>f&&(d=f),0===d)break e;i.set(n.subarray(a,a+d),o),s-=d,a+=d,f-=d,o+=d,r.length-=d;break}r.mode=12;break;case 17:for(;u<14;){if(0===s)break e;s--,l+=n[a++]<>>=5,u-=5,r.ndist=1+(31&l),l>>>=5,u-=5,r.ncode=4+(15&l),l>>>=4,u-=4,r.nlen>286||r.ndist>30){e.msg="too many length or distance symbols",r.mode=30;break}r.have=0,r.mode=18;case 18:for(;r.have=359?359:i;i-=o;do{t+=e[a++]<<8,r+=t+=e[a++]}while(--o);t=(65535&t)+(t>>>16),r=(65535&r)+(r>>>16)}return 1&n&&(r+=t+=e[a]<<8),((r=(65535&r)+(r>>>16))<<16|(t=(65535&t)+(t>>>16)))>>>0},readHeaderInfo:function(e,t){var r=t.ptr,n=new Uint8Array(e,r,6),i={};if(i.fileIdentifierString=String.fromCharCode.apply(null,n),0!==i.fileIdentifierString.lastIndexOf("Lerc2",0))throw"Unexpected file identifier string (expect Lerc2 ): "+i.fileIdentifierString;r+=6;var a,o=new DataView(e,r,8),s=o.getInt32(0,!0);if(i.fileVersion=s,r+=4,s>=3&&(i.checksum=o.getUint32(4,!0),r+=4),o=new DataView(e,r,12),i.height=o.getUint32(0,!0),i.width=o.getUint32(4,!0),r+=8,s>=4?(i.numDims=o.getUint32(8,!0),r+=4):i.numDims=1,o=new DataView(e,r,40),i.numValidPixel=o.getUint32(0,!0),i.microBlockSize=o.getInt32(4,!0),i.blobSize=o.getInt32(8,!0),i.imageType=o.getInt32(12,!0),i.maxZError=o.getFloat64(16,!0),i.zMin=o.getFloat64(24,!0),i.zMax=o.getFloat64(32,!0),r+=40,t.headerInfo=i,t.ptr=r,s>=3&&(a=s>=4?52:48,this.computeChecksumFletcher32(new Uint8Array(e,r-a,i.blobSize-14))!==i.checksum))throw"Checksum failed.";return!0},checkMinMaxRanges:function(e,t){var r=t.headerInfo,n=this.getDataTypeArray(r.imageType),i=r.numDims*this.getDataTypeSize(r.imageType),a=this.readSubArray(e,t.ptr,n,i),o=this.readSubArray(e,t.ptr+i,n,i);t.ptr+=2*i;var s,f=!0;for(s=0;s1&&!n&&(t.pixels.resultPixels=h.swapDimensionOrder(t.pixels.resultPixels,s,j,r))},formatFileInfo:function(e){return{fileIdentifierString:e.headerInfo.fileIdentifierString,fileVersion:e.headerInfo.fileVersion,imageType:e.headerInfo.imageType,height:e.headerInfo.height,width:e.headerInfo.width,numValidPixel:e.headerInfo.numValidPixel,microBlockSize:e.headerInfo.microBlockSize,blobSize:e.headerInfo.blobSize,maxZError:e.headerInfo.maxZError,pixelType:h.getPixelType(e.headerInfo.imageType),eofOffset:e.eofOffset,mask:e.mask?{numBytes:e.mask.numBytes}:null,pixels:{numBlocksX:e.pixels.numBlocksX,numBlocksY:e.pixels.numBlocksY,maxValue:e.headerInfo.zMax,minValue:e.headerInfo.zMin,noDataValue:e.noDataValue}}},constructConstantSurface:function(e,t){var r=e.headerInfo.zMax,n=e.headerInfo.zMin,i=e.headerInfo.maxValues,a=e.headerInfo.numDims,o=e.headerInfo.height*e.headerInfo.width,s=0,f=0,l=0,u=e.pixels.resultMask,c=e.pixels.resultPixels;if(u)if(a>1){if(t)for(s=0;s1&&n!==r)if(t)for(s=0;s=-128&&t<=127;break;case 1:r=t>=0&&t<=255;break;case 2:r=t>=-32768&&t<=32767;break;case 3:r=t>=0&&t<=65536;break;case 4:r=t>=-2147483648&&t<=2147483647;break;case 5:r=t>=0&&t<=4294967296;break;case 6:r=t>=-34027999387901484e22&&t<=34027999387901484e22;break;case 7:r=t>=-17976931348623157e292&&t<=17976931348623157e292;break;default:r=!1}return r},getDataTypeSize:function(e){var t=0;switch(e){case 0:case 1:t=1;break;case 2:case 3:t=2;break;case 4:case 5:case 6:t=4;break;case 7:t=8;break;default:t=e}return t},getDataTypeUsed:function(e,t){var r=e;switch(e){case 2:case 4:r=e-t;break;case 3:case 5:r=e-2*t;break;case 6:r=0===t?e:1===t?2:1;break;case 7:r=0===t?e:e-2*t+1;break;default:r=e}return r},getOnePixel:function(e,t,r,n){var i=0;switch(r){case 0:i=n.getInt8(t);break;case 1:i=n.getUint8(t);break;case 2:i=n.getInt16(t,!0);break;case 3:i=n.getUint16(t,!0);break;case 4:i=n.getInt32(t,!0);break;case 5:i=n.getUInt32(t,!0);break;case 6:i=n.getFloat32(t,!0);break;case 7:i=n.getFloat64(t,!0);break;default:throw"the decoder does not understand this pixel type"}return i},swapDimensionOrder:function(e,t,r,n,i){var a=0,o=0,s=0,f=0,l=e;if(r>1)if(l=new n(t*r),i)for(a=0;a
Recently, several morphologies, each with its advantages, have been proposed for the GelSight high-resolution tactile sensors. However, existing simulation methods are limited to flat-surface sensors, which prevents the usage of the newer sensors with non-flat morphologies in Sim2Real experiments. In this work, we extend our previously proposed GelSight simulation method, which was developed for flat-surface sensors, and propose a novel method for curved sensors. Particularly we address the simulation of light rays travelling through the curved tactile membrane in the form of geodesic paths. The method is validated by simulating our finger-shaped GelTip[1] sensor and comparing the generated synthetic tactile images against the corresponding real images. Our extensive experiments show that combining the illumination generated from the geodesic paths, with a background image from the real sensor, produces the best results when compared to the lighting generated by direct linear paths in the same conditions. As the method is parameterized by the sensor mesh, it can be applied, in principle, to simulate a tactile sensor of any morphology. The proposed method not only unlocks simulating existing optical tactile sensors of complex morphologies but also enables experimenting with sensors of novel morphologies, before the fabrication of the real sensor.
Starting from our previous simulation method for flat GelSight sensors[2] additional steps are considered to address sensors of curved geometry, where the light travels through its curved membrane. Particularly, we address the sensor internal illumination, that for flat sensors is a constant vector field, starting on the LEDs / membrane edge and being tangent to the flat surface. For curved sensors, we consider the geodesic, i.e., the shortest path on a curved surface, as a reasonable path for the light to travel in the membrane. Four alternatives are considered for computing the light field: Linear, Plane, Geodesic and Transport. Linear is a baseline radial light field, originating from a point light source. Plane is a naïve approach for computing the geodesic path, based on the intersection of a plane that passes through the light source, with the sensor mesh. Geodesic and Transport are publicly available implementations for the computation of geodesic paths and vector transport over meshes (Pygeodesic and Transport respectively). Since the light within the entire sensor is constant, we can pre-compute it beforehand. This way, the method can be run online. The overall approach consists of two steps: light fields are firstly computed offline; then, using the computed light fields, the simulation model is run online for each frame of the sensor simulation. The online simulation model consists of three main steps: 1) smoothing of the raw depth map, captured from the simulation environment, to mimic the elastic deformation of the real sensor; 2) mapping of the smoothed depth map onto a point-cloud in the camera coordinates frame; 3) generation of the tactile image using Phong’s reflection model and the pre-computed light fields.
To evaluate our method, we collect aligned datasets of RGB tactile image with the real setup and depth-maps with the corresponding simulation, by tapping 8 3D printed objects[3] against the sensor, as shown in Figure 1. The data collection motions consist of equidistant and perpendicular taps on 3 straight paths along the longer axis of the sensor, from the tip to its base. The printers and sensors are controlled sing Yarok[2], ensuring repeatability between environments.
Quantitative analysis. For each light field, we generate the corresponding version of the synthetic RGB dataset, and compare it against real aligned correspondences, as shown in Figure 4. As reported in the paper, the Linear light field results in a better Structural Similarity (SSIM) of 0.84, when no ambient light is considered. However, if either a solid ambient illumination or the background image from the real sensor is used, then the Plane, Geodesic and Transport methods produce the best results, with a high SSIM of 0.93. This directly comes from the fact that the Plane, Geodesic and Transport methods do not produce any illumination in areas that are not being deformed by a contact, which contrasts with the Linear method that generates bright gradients throughout the entire sensor surface, as highlighted in Figure 3.
Sim2Real transfer. The real and simulation RGB datasets (using the geodesic light-field) are split into equal training (80%) and validation (20%) splits, and a ResNet is optimized. We find that, in general, performing these tasks with the simulated images (Sim2Sim) results in better results than the corresponding Real2Real cases. One possible reason could be that with the real sensor, the contact forces between the in-contact object and the sensor result in some bending of the sensor membrane and consecutive weaker imprints, as noticeable in the examples shown in Figure 4. On the other hand, while the Extended dataset and cropping of centred patches positively contribute to a reduction in the Sim2Real gap, the Sim2Real gap is still significant: with the model trained with the simulated data achieving only a maximum classification accuracy of 50.0%, in the classification task, versus the 64.3% obtained by the model trained with the real data. In the Sim2Real contact localisation, the model obtains a minimum localisation error of 11.8mm, worse than the 8.4mm obtained by the model trained with real data.
We proposed a novel approach for simulating GelSight sensors of curved geometry, such as the GelTip. The specific considerations of the light trajectories within the tactile membrane help us in better verify whether our assumptions about the real sensor are true. For instance, with the analysis of different types of light fields, we verify that tactile images captured by the existing GelTip sensors contain a high degree of light that does not travel parallel to the sensor surface, as idealised by the early GelSight working principle. In the future, we will compare the tactile images obtained from a real sensor fabricated using the morphology design optimised in the simulation, and also apply our proposed simulation model in Sim2Real learning for tasks like robot grasping and manipulation with optical tactile sensing.
To address the challenging problem of grasping objects in cluttered environments, recent works have proposed training Deep Learning models to predict the success of candidate grasps, from large datasets of monocular images that are collected and annotated fully autonomously. A smaller group of works have also proposed video predictive models to address non-prehensile tasks. However, none have yet combined these two strategies for addressing the aforementioned grasping problem. In this work, we investigate and compare a self-supervised model-free approach, to estimate the success of a candidate grasp, against the model-based counterpart that exploits a self-supervised predictive model that hallucinates a close-up observation of the gripper when about to grasp an object. This decomposition allows us to split the training process into two distinct phases (future observation predictor and grasp success estimator) and, with that, to exploit more data for supervision, without requiring any additional data annotation, as well as analysing each stage individually. Our experiments demonstrate that despite a model-free ConvNet quickly overfitting to the training dataset, a similar network is able to discriminate failed from successful grasps when the image captured right before the grasp occurs, is considered. Further, our predictive model is able to hallucinate these future observations without requiring a calibrated camera or additional annotations. This work contributes to further highlighting the advantage of task decomposition, and the exploitation of predictive models and unsupervised learning in robotics contexts, where labelling datasets extensively is often not feasible.
To evaluate our models, we set up a robotic actuator to collect randomly sampled grasps for 4 days. In total, 24,364 random grasps are collected: 3,789 (15.6%) successful grasps, 16,306 (66.9%) aborted attempts, i.e., the gripper collided against an object while moving, and 4,269 (17.6%) empty grasps. For each grasp, we capture images of the scene for three distinct moments Ib, Id and Ia, as illustrated in Algorithm 1. Since a stereo camera is used, effectively two frames are stored per moment. In this work, we use the two frames to double the dataset size.
The development of Robotics Software is a combersome task, often requiring the programming of multiple components for handling different sensors and actuators. Fortunately the adoption of software frameworks, namely the Robotic Operating System (ROS), has enabled the sharing of these components amongst practitioners, resulting in quicker prototpyings. Nonetheless, ROS is a complex framework, with a steep learning curve, requiring the understanding of the framework achitecture, messaging system, multiple programming languages, highly tied the operating system, etc. while, not supporting out-of-the-box more sophisticated simulators, such as MuJoCo – of high interest to current Machine Learning/Robotics research. On the other hand, MuJoCo, which can be set-up and run from a single Python script, does not include many capabilities for modularity or integration with real world hardware. To address this gap, we introduce the Yarok framework. Using Yarok practitioners can describe components using an slightly extended MuJoCo markup language, that are directly represented as Python classes, and have full experimental setups in a few clear lines of code. Further, the low level interfaces, for each component, for interacting with the environment (MuJoCo, or real world), can be also written as Python classes, and are enabled by the framework, for the corresponding environment.
The central concept in Yarok is the component. A Component is a Python annotated class that allows you to bundle together the component controller and template description (using extended MuJoCo MJCF). In the component, you can also define environment interfaces that, at runtime, depending on what enviroment yarok is running, override the environment-independent controller class.
1 |
Yarok allows you to easily compose complex robots using components of sensors and actuators. Bellow, we show how o include a custom gripper component onto your robot model, as well as nesting a sensor onto the gripper finger. Further, you’re able to request the corresponding gripper and sensor instance controllers to be injected onto the robot controller.
1 | from my_shared_components import MyGripper, MyTouchSensor |
Similarly to xacro in the Robotic Operating System (ROS), Yarok adds config variables and if/for statements to templates. See below how we defined a parameterizable tumble tower of blocks using these macros. Define the variables in the defaults section, and override it when running the platform.
1 | <for each='range(rows)' as='z'> |
1 | from yarok import interface |
Once you have your components tree defined, you just wrap it onto a Platform object and run it. By default, Yarok runs on the simulation environment, loading MuJoCo interfaces, but you can also configure it to run on “real” environment, and it will load the real interfaces. This way, you can write your behaviours once and run them both in simulation and the real world.
1 | from yarok import Platform, PlatformMJC |
1 | <body euler="0 0 1.57" pos="-0.15 0.4 0.3"> |
In this post we have just give you a quick glance at what you can do with Yarok, if you would like to learn more and getting started using it, visit the project repository and start experimenting.
]]>Most current works in Sim2Real learning for robotic manipulation tasks leverage camera vision that may be significantly occluded by robot hands during the manipulation. Tactile sensing offers complementary information to vision and can compensate for the information loss caused by the occlusion. However, the use of tactile sensing is restricted in the Sim2Real research due to no simulated tactile sensors being available. To mitigate the gap, we introduce a novel approach for simulating a GelSight tactile sensor in the commonly used Gazebo simulator. Similar to the real GelSight sensor, the simulated sensor can produce high-resolution images by an optical sensor from the interaction between the touched object and an opaque soft membrane. It can indirectly sense forces, geometry, texture and other properties of the object and enables Sim2Real learning with tactile sensing. Preliminary experimental results have shown that the simulated sensor could generate realistic outputs similar to the ones captured by a real GelSight sensor.
This GelSight Simulation method was firstly proposed at the 2019 ICRA ViTac Worshop and, given the interest shown by the community, now revised and extended in a new publication[1]. For instance, the initial elastomer deformation approximation approach, generated unrealistic sharp contouring around the in-contact areas, as shown in Figure 2. Improvements achieved now, are shown in Figure 3.
To produce the necessary real datasets, a GelSight sensor is mounted onto a Fused Deposition Modeling (FDM) 3D printer A30 from Geeetech. A set of objects with different shapes on the top is designed and 3D printed using the Form 2 Stereolithography (SLA) 3D printer. A Virtual World comprised of a FDM printer, a GelSight sensor and a set of virtual objects, is also set up. Identical real and virtual datasets are then collected.
These include the STL files for printing the 21 set of objects and support mount,
the raw real and virtual datasets, and the aligned datasets using the per-object alignment method.
Please refer to the paper for more details about the experiments.
One aspect to consider in the Sim2Real learning is the Sim2Real gap that results from characteristics of the real world being not modeled in the simulation. In our case, we find that one major difference between the real and synthetic samples are the textures introduced by the 3D printing process. To mitigate this issue, we create twelve texture maps using GIMP that resemble the textures observed in the real samples, as shown in Figure 1. By randomly perturbing the captured virtual depth-maps with such textures, we are able to produce an effective data augmentation scheme that significantly improves the Sim2Real transition, from a 43.76% classification accuracy to 76.19%, in the real data.
Sensing contacts throughout the entire finger is an highly valuable capability for a robots (and humans) when carrying manipulation tasks, as vision-based sensing often suffers from occlusions or inaccurate estimations. Current tactile sensors suffer from one of two drawbacks: low resolution readings, or a limited contact measurement area. We propose a finger-shaped optical sensor that has the shape of a finger and can sense contacts on any location of its surface. Our experiments show that the sensor can effectively capture such contacts throughout its surface.
We derive the protective function that maps pixels in the image space into points on the sensor surface, as illustrated in the picture below. The two shown rays intersect the sensor surface in the spherical region, in red, and the cylindrical region, in blue; and each ray intersects three relevant points: the frame of reference origin, a point in the sensor surface and the corresponding projected point in the image plane.
We evaluate our GelTip sensor from on two tasks. Firstly, we demonstrate that the tactile images captured by our sensor can be used to achieve contact localization. Secondly, we demonstrate the advantages of considering touch sensing in manipulation tasks, such as grasping.
Two GelTip sensors are installed on a robotic actuator and a 3D printed mount that holds a small 3D printed object placed on top of a wooden block. The actuator moves in small increments and collects tactile images annotated with the known contact positions. We then implement an image-subtraction based algorithm to localise such contacts in image space. Then, using the projective model the localised contact points are projected into world coordinates. The errors between the true and predicted positions, measured as the Euclidean distance, are then assessed.
To demonstrate the potential of all-around sensing, in the context of manipulation/grasping tasks a Blocks World is carried. The robot actuator moves row by row, attempting to grasp each block. Two different policies are analysed: one in which the robot grasps each block randomly, and another in which touch feedback is used to adjust the initially random grasp (the video above shows the latter). The experiment shows that when using touch feedback the robot grasps all the blocks successfully, even with the initial uncertainty.
For more information about the GelTip sensor, its fabrication process, and the executed experiments, checkout the papers referenced bellow.
Developing autonomous assistants to help with domestic tasks is a vital topic in robotics research. Among these tasks, garment folding is one of them that is still far from being achieved mainly due to the large number of possible configurations that a crumpled piece of clothing may exhibit. Research has been done on either estimating the pose of the garment as a whole or detecting the landmarks for grasping separately. However, such works constrain the capability of the robots to perceive the states of the garment by limiting the representations for one single task. In this work, we propose a novel end-to-end deep learning model named GarmNet that is able to simultaneously localize the garment and detect landmarks for grasping. The localization of the garment represents the global information for recognising the category of the garment, whereas the detection of landmarks can facilitate sub-sequent grasping actions. We train and evaluate our proposed GarmNet model using the CloPeMa Garment dataset that contains 3,330 images of different garment types in different poses. The experiments show that the inclusion of landmark detection (GarmNet-B) can largely improve the garment localization, with an error rate of 24.7% lower. Approaches as ours are important for robotics applications, as these offer scalable to many classes, memory and processing efficient solutions.
So, why study this problem? The central motivation lies in automating the folding of clothing items, an activity that is present throughout our society and either consumes our personal time or is reflected in the cost of services or products we purchase. Any system we can imagine for automating this task will undoubtedly require a clothing recognition module, which is the focus of this research.
Furthermore, when considering clothing recognition in isolation, it becomes evident that this capability is not only useful for folding clothes but also for other systems. For example, intelligent surveillance systems may want to locate a person in a video feed based on their clothing description, or e-commerce recommendation systems may need to provide suggestions based solely on information available in a photograph. By developing a reliable and efficient model for clothing recognition, we aim to contribute to the advancement of these technologies and improve their performance in various applications.
Given an image containing a clothing item in a semi-crumpled or semi-folded pose, we aim to develop a system capable of classifying and localizing the clothing item with a bounding box. Additionally, the system should be able to detect and classify each associated keypoint of interest on the clothing item, such as the outer lower corner of the right leg or the position between the legs. By tackling this problem, we hope to create a robust solution that can be integrated into various applications, ultimately improving their efficiency and effectiveness in recognizing and processing clothing items.
Our model was initially inspired by the R-CNN architecture. However, after undergoing several modifications, it has also come to resemble other models such as YOLO and DeepFashion. Similar to YOLO, our model performs object detection in a single step. It shares some aspects with DeepFashion by using keypoint predictions to enhance the overall image feature prediction, ultimately improving the classification of the object. Our model also has its unique characteristics, such as incorporating an extra class for background segmentation. We can break down the model into three primary modules: Feature Extraction, and two additional branches. One branch is responsible for determining the class and bounding box of the clothing item, while the other branch focuses on identifying the keypoints of interest. This combination of elements allows our model to efficiently analyze images and deliver accurate results.
The feature extractor in our model is implemented using a 50-layer ResNet, which has been pre-trained on datasets provided by the ImageNet platform. From this model, we have chosen two output points. The first one produces a 14x14 volume, which we use as input for the keypoint detection module. The second output point generates a vector with 2048 features, which we use as input for the clothing item localization and classification module. The set of features computed between the first and second output points can be interpreted as global features of the image. This allows our model to effectively analyze the overall structure and content of the image, contributing to its accuracy and efficiency in detecting objects and keypoints.
The clothing item localization and classification module consists of an intermediate dense operation and two outputs. One output performs the bounding box regression for the clothing item, while the other predicts the clothing item’s class, encoded in a one-hot format. The regression output is trained using the mean squared error loss function, while the classification output is trained using the cross-entropy loss function, applied in conjunction with the softmax activation.
The keypoint detector can be interpreted as applying a small localizer for each spatial position of the feature extractor’s output volume, implemented with convolutional operations. In other words, the intermediate layer has a 3x3 filter and 1024 activation maps, which is equivalent to applying a dense operation with an output vector of 1024 features. Subsequently, we apply the dense operation to each of these feature vectors using a convolutional operation with a 1x1 filter. This approach enables our model to effectively detect keypoints of interest on the clothing items within the image.
We train each branch of the model separately during 40 iterations, batch size of 30 and Adadelta optimizer. The Garment localizer achieves: 100% accuracy on classification and 43% classification+localization. The Landmark detector achieves: 37.8% mAP.
We have shown a possible way to perform landmark detection together with garment localization, based on a object detection approach. We should notice, though, the fact that for some landmark classes the AP is 0% i.e., no predictions are made. This, together with the fact that many hyper-parameters can be tweaked and the dataset is not very large, hints to the fact that better results could be obtained. To attempt to tackle more complex problems, different from simple classification and/or regression, it is important to understand each type of module, and the gradient descent dynamics. When implementing custom losses or layers, the vectorial paradigm is always present. Better tools for the development of such systems can be envisioned, that would contribute to quicker design and implementation iterations. Instead of attempting to extract all the landmarks and garment class+localization in one step, other approaches should be considered. For instance, given an image what’s the best point to grasp? Or, how much folded is the present garment piece?
Working with experimental data requires reading from and writing to multiple data formats, with researchers commonly having to write custom scripts to convert data from one format to another. However, and as discussed in our “e-WindLidar: making wind lidar data FAIR” report, at the right level of abstraction a common data model can be defined to describe all measured data. As such, we propose a modular data converter that enables setting up readers and writers in an any-to-any fashion, and can be configured to attach all sorts of metadata contributing to making its output data more FAIR.
LIDACO (LIdar DAta COnverter) is a Python library and executable that enables a modular writing of FAIR lidar data converters for various types of lidars. As such, you can use it directly from the terminal, or include it in your projects. When using it a configuration (file) must be specified, which includes information about the Reader used to import the data from input files and the Writer used to write the output file(s). Lidaco configuration system is hierarchical to motivate the configurations reusability. As such, each configuration file can import other configuration files. With this mechanism you can to split your configurations into devices, scenarios, campaigns and so on. And associate the appropriate metadata with at the corresponding level.
Lidaco works on datasets that can be described using the unidata Common Data Model and e-WindLidar standard. It can be used to process single files or entire folders. Currently, Lidaco supports out of the box input data recorded by: Galion, WLS70, Windcube v1, Windcube v2, Windcube 100s, 200s and 400s, Long-range WindScanner and ZephIR300; and can export MetadataCard, NcML and netCDF4. However, given its modular design, users can write their on reader/writers. The converter and detail description on how to use it together with practical examples can be found at the e-WindLidar public GitHub repository. Check the link on the top of the page.
]]>Experimental field campaigns for collecting wind data are essential for academic research and the wind energy industry, but challenging to organize due to the complex equipment and infrastructure required. Existing e-Science platforms have been developed for more generic domains, which prevents them from capturing the details and requirements of the field. In this project we have developed WindsPT: an e-Science platform for planning, executing, and disseminating wind measurement campaign data. Additionally, we propose a decentralized protocol for transferring large volumes of data from the in-site devices to our platform, ensuring data replication. With an easy-to-use Web interface, WindsPT promotes collaboration between participants, disseminates results among the stakeholders, publishes metadata, uses DOI, and includes metadata that enables machine-to-machine communication.
The platform development started with the Perdigão 2017[1] campaign and the need of gathering partners collaboration intentions for the campaign planning, as well as producing a reference platform for gathering all the campaign related information (planning documents, datasets, publications, etc). A two-phase questionnaire was developed and made available to the representatives of each institution that would be participating in the campaign. The questionnaire results became the starting point for in-depth discussions among institution representatives, campaign managers, and the development team, with the goal of establishing a common vocabulary amongst all involved parties. Particularly, data curators pruned, and improved with additional metadata, the categories of equipment and the equipment, that was collaboratively added to the platform through the surveys. This became the publicly available equipment catalog. Other discussions revolved around the definition of a measurement station and how to model it in the platform. With that, campaign managers geo-registered in the platform the planned measurement stations.
The campaign profile fulfils the second function of the platform — to host all information related to the campaign. Here, users can find all information related to the campaign: planning and dissemination documents, gathered datasets, measurement equipment and stations (including their geo-registration and operating timelines), participants (visiting timeline and profile) and a logbook that can be used to register all relevant events.
WindsPT was built with an Angular frontend and NodeJS/ExpressJS backend that exposes a REST api. Relational data is stored on a PostgresSQL database, and the Sequelize ORM is used for handling database queries. To store the planning and dissemination documents, MongoDB GridFS is used under the ExpressJS backend. Given the potentially huge (and unbounded) size of the datasets, a decentralized protocol has been established between data producer institutions, that consists on using Linux rsync protocol periodically between peers of the pool, ensuring the data is replicated across all, while minimising data transfer between nodes. In one of these nodes (ours), a THREDDS Data Server (TDS) gathers files and folders reorganises them in catalogues, and makes them available through different protocols (HTTP, OpeNDAP OGC WMS, etc). The WindsPT backend then re-exposes this catalogued data through its rest api to create the corresponding graphical user interface. This allows us to add additional features such as the capability to download “folders” as zip files, and make the interface similar to the “Documents” interface.
The WindsPT platform as proved to be a valuable tool during all the phases of the Perdigão 2017 large field experiment. Following that, a multi-campaign version of WindsPT[2] platform has been designed to host multiple campaigns on multiple research projects. The advantage of such multi-campaign platform, is that, this way, information about equipment and researchers can be shared across campaigns. Full-text search capabilities, have been added enabling users to find any information across the entire pool of campaigns with just one search query.
This is going to be a very atypical post on this blog but, because I spent almost two weeks to figure it out I’ll leave it here. These configs will probably change with time and so, I’ll be updating this post in a logbook manner.
I’m running Antergos with Gnome desktop environment and 4.7.6-1-ARCH kernel on UEFI, gpt/lvm partitioning and systemd-boot loader.
If you have any problems/suggestions, please leave comment below.
Good luck.
]]>Since my teenager years I’ve been using the internet quite a lot and, with that, learning many on many different topics. Such was only possible because of the large amount of educational content provided by “anonymous” people through blogs, forums and video channels. Probably because of that, I always felt admired by this spirit of free and open sharing and mutual help.
From early on, I’ve found myself thinking about creating a blog or a video channel about “How to program” or something similar, that would allow me to give back what I’ve learned, but I always came across other great content providers that were making it much better than I would probably ever done.
Today with my studies course reaching its end, my master thesis arriving and my attention focusing on Computer Vision, Artificial Intelligence and Robotics I feel that the uniqueness of information that I would be capable to produce is greater and time is right to, once for all, create such web page. Also, this will be most valuable for me by serving as an alive showcase of what i’m doing and my interests as well as an extra motivation to learn and explore more. (You may find that this is not a novelty on my life by reading my About page.)
So, in this blog you may expect to find content about AI, CV, Robotics and other computer related topics. These could appear in two types of posts: experiments/toy projects or supper summaries of “complex” subjects that I find possible to compress without lose to much. It is also possible that I explore here some of my philosophical ideas that often lie between natural intelligence/consciousnesses and the artificial counterpart. Finally, I may be sharing here some of the 5 minutes ideas that I have through the day and that I would like to work on if I had the number of lives of a cat.
That’s all; Thanks!
Until the next post.