WIP: AR support

ipyvolume/ui.py

@@ -25,6 +25,7 @@ class Container(v.VuetifyTemplate):
     children = traitlets.List().tag(sync=True, **widgets.widget_serialization)
     models = traitlets.Any({'figure': {}}).tag(sync=True)
     panels = traitlets.List(traitlets.CInt(), default_value=[0, 1, 2]).tag(sync=True)
+    ar_supported = traitlets.Bool(False).tag(sync=True)
 
 
 class Popup(v.VuetifyTemplate):

ipyvolume/vue/container.vue

@@ -29,6 +29,12 @@
                     </v-expansion-panel-content>
                 </v-expansion-panel>
             </v-expansion-panels>
+            <div v-if="ar_supported">
+                <v-btn @click="startAR" outlined>
+                    <v-icon left>mdi-glasses</v-icon>
+                    start ar
+                </v-btn>
+            </div>
         </div>
     </div>
 </template>
@@ -44,13 +50,17 @@
 
 <script>
     module.export = {
-        created() {
+        async created() {
             console.log('created', this.$refs)
+            if ('xr' in navigator) {
+                const supported = await navigator.xr.isSessionSupported('immersive-ar')
+                this.ar_supported = supported;
+            }
         },
         mounted() {
             const figureComponent = this.$refs.figure;
             (async () => {
-                const figure = await this.viewCtx.getModelById(this.figure.substr(10));
+                const figure = await this.figureModel;
                 function bbproxy(model, attrs, widgetAttrs) {
                     const proxy = {}
 
@@ -109,7 +119,18 @@
                 this.$set(this.models, 'figure', bbproxy(figure, ["show"]));
             })();
         },
+        computed: {
+            figureModel() {
+                return this.viewCtx.getModelById(this.figure.substr(10))
+            }
+        },
         methods: {
+            getFigureModel() {
+                return
+            },
+            async startAR() {
+                (await this.figureModel).set('start_ar', true);
+            }
         }
     }
 

ipyvolume/widgets.py

@@ -358,6 +358,8 @@ class Figure(ipywebrtc.MediaStream):
         pythreejs.Camera, allow_none=True, help='A :any:`pythreejs.Camera` instance to control the camera'
     ).tag(sync=True, **widgets.widget_serialization)
 
+    start_ar = traitlets.Bool(False).tag(sync=True)
+
     @traitlets.default('camera')
     def _default_camera(self):
         # see https://github.com/maartenbreddels/ipyvolume/pull/40 for an explanation

js/src/ar.js

@@ -1,203 +1,106 @@
-import { WebXRManager } from 'three'
-
-export function createButton( renderer, onStart, onEnd, sessionInit = {} ) {
-
-    const button = document.createElement( 'button' );
-    console.log('__renderer:', renderer)
-
-    function showStartAR( /*device*/ ) {
-
-        if ( sessionInit.domOverlay === undefined ) {
-
-            var overlay = document.createElement( 'div' );
-            overlay.id = 'aroverlay';
-            overlay.style.display = 'none';
-            document.body.appendChild( overlay );
-            // document.document.getElementById('bliep').appendChild( overlay );
-
-            var svg = document.createElementNS( 'http://www.w3.org/2000/svg', 'svg' );
-            svg.setAttribute( 'width', 38 );
-            svg.setAttribute( 'height', 38 );
-            svg.style.position = 'absolute';
-            svg.style.right = '20px';
-            svg.style.top = '20px';
-            svg.addEventListener( 'click', function () {
-                onEnd();
-                console.log('end');
-                currentSession.end();
-
-            } );
-            overlay.appendChild( svg );
-
-            var path = document.createElementNS( 'http://www.w3.org/2000/svg', 'path' );
-            path.setAttribute( 'd', 'M 12,12 L 28,28 M 28,12 12,28' );
-            path.setAttribute( 'stroke', '#fff' );
-            path.setAttribute( 'stroke-width', 2 );
-            svg.appendChild( path );
-
-            if ( sessionInit.optionalFeatures === undefined ) {
-
-                sessionInit.optionalFeatures = [];
-
-            }
-
-            sessionInit.optionalFeatures.push( 'dom-overlay' );
-            sessionInit.domOverlay = { root: overlay };
-
-        }
-
-        //
-
-        let currentSession = null;
-
-        async function onSessionStarted( session ) {
-            session.addEventListener( 'end', onSessionEnded );
-            console.log('rr', renderer);
-            console.log('start session', renderer.xr)
-            renderer.xr.setReferenceSpaceType( 'local' );
-            console.log('b4 set session');
-            await renderer.xr.setSession( session );
-            renderer.setSize( window.innerWidth, window.innerHeight );
-            console.log('after set session');
-            button.textContent = 'STOP AR';
-            sessionInit.domOverlay.root.style.display = '';
-            sessionInit.domOverlay.root.style.display = '';
-
-            currentSession = session;
-
-        }
-
-        function onSessionEnded( /*event*/ ) {
-
-            currentSession.removeEventListener( 'end', onSessionEnded );
-
-            button.textContent = 'START AR';
-            sessionInit.domOverlay.root.style.display = 'none';
-
-            currentSession = null;
-
-        }
-
-        //
-
-        button.style.display = '';
-        button.id = 'arbtn';
-
-        button.style.cursor = 'pointer';
-        button.style.left = 'calc(50% - 50px)';
-        button.style.width = '100px';
-
-        button.textContent = 'START AR';
-
-        button.onmouseenter = function () {
-
-            button.style.opacity = '1.0';
-
-        };
-
-        button.onmouseleave = function () {
-
-            button.style.opacity = '0.5';
-
-        };
-
-        button.onclick = function () {
-            console.log('current', currentSession);
-            if ( currentSession === null ) {
-                console.log('requesting session')
-                onStart();
-                navigator.xr.requestSession( 'immersive-ar', sessionInit ).then( onSessionStarted );
-
-            } else {
+export const closeOverlay = document.createElement("div");
+closeOverlay.style.display = "none";
+document.body.appendChild(closeOverlay);
+
+let onClose = () => console.log("onClose not set");
+
+const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
+svg.setAttribute("width", 38);
+svg.setAttribute("height", 38);
+svg.style.position = "absolute";
+svg.style.right = "20px";
+svg.style.top = "20px";
+svg.addEventListener("click", () => onClose());
+closeOverlay.appendChild(svg);
+
+const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
+path.setAttribute("d", "M 12,12 L 28,28 M 28,12 12,28");
+path.setAttribute("stroke", "#fff");
+path.setAttribute("stroke-width", 2);
+svg.appendChild(path);
+
+function setOnClose(fn) {
+    onClose = fn;
+}
 
-                currentSession.end();
+function showCloseOverlay(value) {
+    closeOverlay.style.display = value ? "" : "none";
+}
 
-            }
+let lastXrTransform = null;
+let lastWidth = null;
+let lastHeight = null;
 
-        };
+export function createController(renderer, scene, onSelect) {
+    const arController = renderer.xr.getController(0);
+    arController.addEventListener("select", () => onSelect(lastXrTransform));
+    scene.add(arController);
+    return arController;
+}
 
+export async function start(renderer, renderFn, onStop) {
+    if (renderer.xr.getSession() !== null) {
+        return;
     }
+    renderer.xr.enabled = true;
 
-    function disableButton() {
+    const sessionInit = {
+        optionalFeatures: ["dom-overlay", "hit-test"],
+        domOverlay: { root: closeOverlay }
+    };
 
-        button.style.display = '';
+    showCloseOverlay(true);
 
-        button.style.cursor = 'auto';
-        button.style.left = 'calc(50% - 75px)';
-        button.style.width = '150px';
+    const session = await navigator.xr.requestSession("immersive-ar", sessionInit);
+    renderer.xr.setReferenceSpaceType("local");
+    renderer.xr.setSession(session);
+    const { x, y } = renderer.getSize();
+    lastWidth = x;
+    lastHeight = y;
+    renderer.setSize(window.innerWidth, window.innerHeight);
 
-        button.onmouseenter = null;
-        button.onmouseleave = null;
+    lastXrTransform = null;
 
-        button.onclick = null;
+    renderer.setAnimationLoop(
+        await extractHitTransform(
+            renderer,
+            renderFn,
+            (xrTransform) => { lastXrTransform = xrTransform; }));
 
-    }
-
-    function showARNotSupported() {
-        console.log('ar not supported');
-        disableButton();
-
-        button.textContent = 'AR NOT SUPPORTED';
-
-    }
-
-    function stylizeElement( element ) {
-
-        element.style.position = 'absolute';
-        element.style.bottom = '20px';
-        element.style.padding = '12px 6px';
-        element.style.border = '1px solid #fff';
-        element.style.borderRadius = '4px';
-        element.style.background = 'rgba(0,0,0,0.1)';
-        element.style.color = '#fff';
-        element.style.font = 'normal 13px sans-serif';
-        element.style.textAlign = 'center';
-        element.style.opacity = '0.5';
-        element.style.outline = 'none';
-        element.style.zIndex = '999';
+    setOnClose(onStop);
+}
 
+export function stop(renderer) {
+    if (renderer.xr.getSession() === null) {
+        return;
     }
-    console.log('xr:', 'xr' in navigator);
-    if ( 'xr' in navigator ) {
-
-        button.id = 'ARButton';
-        // button.style.display = 'none';
-
-        stylizeElement( button );
-
-        navigator.xr.isSessionSupported( 'immersive-ar' ).then( function ( supported ) {
-            console.log('supported!:', supported)
-            console.log('context', renderer.getContext());
-            supported ? showStartAR() : showARNotSupported();
-
-        } ).catch( showARNotSupported );
-
-        return button;
-
-    } else {
-
-        const message = document.createElement( 'a' );
-
-        if ( window.isSecureContext === false ) {
-
-            message.href = document.location.href.replace( /^http:/, 'https:' );
-            message.innerHTML = 'WEBXR NEEDS HTTPS'; // TODO Improve message
-
-        } else {
-
-            message.href = 'https://immersiveweb.dev/';
-            message.innerHTML = 'WEBXR NOT AVAILABLE';
+    renderer.xr.getSession().end();
+    renderer.xr.setSession(null);
+    lastXrTransform = null;
+    showCloseOverlay(false);
+    renderer.setAnimationLoop(() => null);
+    renderer.setSize(lastWidth, lastHeight);
+}
 
+async function extractHitTransform(renderer, renderFn, onXrTransformChange) {
+    const session = renderer.xr.getSession();
+    const referenceSpace = await session.requestReferenceSpace("viewer");
+    const hitTestSource = await session.requestHitTestSource({ space: referenceSpace });
+
+    return (time, frame) => {
+        if (frame) {
+            // based on https://threejs.org/examples/webxr_ar_hittest.html
+            const referenceSpace = renderer.xr.getReferenceSpace();
+
+            if (hitTestSource) {
+                const hitTestResults = frame.getHitTestResults(hitTestSource);
+                if (hitTestResults.length) {
+                    const hit = hitTestResults[0];
+                    const pose = hit.getPose(referenceSpace);
+                    onXrTransformChange(pose.transform);
+                }
+            }
         }
-
-        message.style.left = 'calc(50% - 90px)';
-        message.style.width = '180px';
-        message.style.textDecoration = 'none';
-
-        stylizeElement( message );
-
-        return message;
-
-    }
-
+        renderFn(time, frame);
+    };
 }