Add ai-model component for text to 3D model generation

aframevr · Mar 21, 2024 · 47e658b · 47e658b
1 parent bb1e7a4
commit 47e658b
Show file tree

Hide file tree

Showing 9 changed files with 101 additions and 3 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -68,6 +68,13 @@
         "ecmaVersion": 8
       }
     },
+    {
+      /* This module uses ES8 async / await due to WebXR Anchor Module integration */
+      "files": ["./src/components/ai-model.js"],
+      "parserOptions": {
+        "ecmaVersion": 8
+      }
+    },
     {
       /* This module uses ES6 for…of loops, and so is parsed as ES6 to avoid errors. */
       "files": ["./src/components/hand-tracking-controls.js"],

diff --git a/examples/showcase/ai-model/index.html b/examples/showcase/ai-model/index.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <title>AI Model • A-Frame</title>
+    <meta name="description" content="Anchor (Mixed Reality) • A-Frame">
+    <script src="../../../dist/aframe-master.js"></script>
+    <script src="../../js/info-message.js"></script>
+  </head>
+  <body>
+    <a-scene
+      info-message="htmlSrc: #messageText">
+      <a-assets>
+        <a-asset-item id="messageText" src="message.html"></a-asset-item>
+      </a-assets>
+      <a-entity ai-model></a-entity>
+    </a-scene>
+  </body>
+</html>
+
diff --git a/examples/showcase/ai-model/message.html b/examples/showcase/ai-model/message.html
@@ -0,0 +1,3 @@
+<p>
+This demo illustrates how to integrate text to 3D in A-Frame</a>
+</p>
diff --git a/package.json b/package.json
@@ -38,6 +38,7 @@
     "buffer": "^6.0.3",
     "debug": "ngokevin/debug#noTimestamp",
     "deep-assign": "^2.0.0",
+    "@gradio/client": "0.14.0",
     "load-bmfont": "^1.2.3",
     "super-animejs": "^3.1.0",
     "super-three": "0.162.0",
@@ -73,6 +74,7 @@
     "karma-webpack": "^5.0.0",
     "markserv": "github:sukima/markserv#feature/fix-broken-websoketio-link",
     "mocha": "^10.0.0",
+    "node-polyfill-webpack-plugin": "^3.0.0",
     "replace-in-file": "^2.5.3",
     "shelljs": "^0.7.7",
     "shx": "^0.2.2",
@@ -115,7 +117,7 @@
     "webxr"
   ],
   "engines": {
-    "node": ">= 4.6.0",
-    "npm": ">= 2.15.9"
+    "node": ">= 21.7.1",
+    "npm": ">= 10.5.0"
   }
 }
diff --git a/src/components/ai-model.js b/src/components/ai-model.js
@@ -0,0 +1,35 @@
+var registerComponent = require('../core/component').registerComponent;
+var GRADIOCLIENT = require('../lib/gradio');
+
+/**
+ * AI model loader.
+ */
+module.exports.Component = registerComponent('ai-model', {
+  schema: {
+    prompt: {type: 'string', default: 'robot'},
+    api: {type: 'string', oneOf: ['perflow']},
+    key: {type: 'string', default: ''}
+  },
+
+  init: function () {
+    this.generateAndRender(this.data.prompt);
+  },
+
+  update: function () {
+
+  },
+
+  generateAndRender: async function generateAndRender (prompt) {
+    var modelEl = document.createElement('a-entity');
+    modelEl.setAttribute('position', '0 1.6 -2');
+    modelEl.setAttribute('rotation', '0 180 0');
+    var app = await GRADIOCLIENT('hansyan/perflow-triposr', {});
+    var result = await app.predict(
+        '/render',
+        (await app.predict('/generate', [prompt, '0'])).data
+    );
+    console.log('Model URL: ' + result.data[0].url);
+    modelEl.setAttribute('obj-model', {obj: result.data[0].url});
+    this.el.sceneEl.appendChild(modelEl);
+  }
+});
diff --git a/src/components/index.js b/src/components/index.js
@@ -1,3 +1,4 @@
+require('./ai-model');
 require('./animation');
 require('./anchored');
 require('./camera');

diff --git a/src/lib/gradio.js b/src/lib/gradio.js
@@ -0,0 +1,3 @@
+var GRADIOCLIENT = require('./gradio.module.js').default;
+
+module.exports = GRADIOCLIENT;
diff --git a/src/lib/gradio.module.js b/src/lib/gradio.module.js
@@ -0,0 +1,5 @@
+import { client } from '@gradio/client';
+
+var GRADIOCLIENT = client;
+
+export default GRADIOCLIENT;
diff --git a/webpack.config.js b/webpack.config.js
@@ -1,5 +1,6 @@
 var path = require('path');
 var webpack = require('webpack');
+var NodePolyfillPlugin = require('node-polyfill-webpack-plugin');
 
 module.exports = {
   entry: './src/index.js',
@@ -29,11 +30,32 @@ module.exports = {
     new webpack.ProvidePlugin({
       process: 'process/browser',
       Buffer: ['buffer', 'Buffer']
-    })
+    }),
+    new webpack.NormalModuleReplacementPlugin(/node:/, (resource) => {
+      resource.request = resource.request.replace(/^node:/, '');
+    }),
+    new NodePolyfillPlugin()
   ],
   resolve: {
     alias: {
       three: 'super-three'
+    },
+    fallback: {
+      // fixes proxy-agent dependencies
+      crypto: false,
+      net: false,
+      dns: false,
+      tls: false,
+      assert: false,
+      http: false,
+      https: false,
+      // fixes next-i18next dependencies
+      path: false,
+      fs: false,
+      // fixes mapbox dependencies
+      events: false,
+      // fixes sentry dependencies
+      process: false
     }
   },
   module: {