vision_paint open ai

SQUASHED: AUTO-COMMIT-demos-openai-vision-paint.md,AUTO-COMMIT-doc-journal-2024-05-23.md-index.md,AUTO-COMMIT-doc-journal-2024-05-23.md-squence-A-B.png,AUTO-COMMIT-src-components-tools-lively-image-editor.html,AUTO-COMMIT-src-components-tools-lively-image-editor.js,AUTO-COMMIT-src-components-tools-openai-audio-chat.js,

demos/openai/vision-paint.md

@@ -0,0 +1,56 @@
+# Paint
+
+
+<script>
+  import OpenAI from "demos/openai/openai.js"
+
+
+  let paint = await (<lively-image-editor id="editor"></lively-image-editor>)
+  paint.clear()
+
+
+  async function view() {
+    let url =  paint.canvas.toDataURL()
+    let blob = await fetch(url).then(r => r.blob())
+    let image = await lively.files.readBlobAsDataURL(blob)
+    let prompt =  {
+            "model": "gpt-4o", 
+            "max_tokens": 500,
+            "temperature": 0.1,
+            "top_p": 1,
+            "n": 1,
+            "stream": false,
+            "stop": "VANILLA",
+            "messages": [
+              { "role": "system", "content": "You are an AI chat bot!" },
+              { "role": "user", "content":  [
+                {
+                  "type": "text",
+                  "text": "What’s in this image?"
+                },
+                {
+                  "type": "image_url",
+                  "image_url": {
+                    "url": image,
+                    "detail": "low" // high
+                  }
+                }
+              ]}
+            ]
+          }
+
+    let json = await OpenAI.openAIRequest(prompt).then(r => r.json())
+    result.textContent = json.choices[0].message.content
+  }
+
+  let result = <div></div>
+
+  let pane = <div>
+    {paint}
+    <button click={() => view()}>view</button> 
+    {result}
+  </div>
+
+
+  pane
+</script>

doc/journal/2024-05-23.md/index.md

@@ -0,0 +1,6 @@
+## 2024-05-23 Vision Paint OpenAI example
+*Author: @JensLincke*
+
+<browse://demos/openai/vision-paint.md>
+
+![](squence-A-B.png)

src/components/tools/lively-image-editor.html

@@ -63,6 +63,11 @@
       height: calc(100% - 35px);
     }
 
+    canvas {
+
+      border: 1px solid gray;
+    }
+
   </style>
    <div id="controls">
     <lively-crayoncolors id="penColor" title="pen"></lively-crayoncolors>

src/components/tools/lively-image-editor.js

@@ -202,6 +202,12 @@ export default class LivelyImageEditor extends Morph {
     return lively.rect(lively.pt(0,0), lively.getExtent(this.canvas)).containsPoint(pos)
   }
 
+  clear() {
+    this.ctx.fillStyle = 'white'; // Set the fill color to blue
+    this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
+  }
+
+
   // #important
   onPointerMove(evt) {
     evt.preventDefault()
@@ -325,16 +331,13 @@ export default class LivelyImageEditor extends Morph {
     const menuElements = [
       ["save", () => this.onSave()],
       ["save as...", () => this.onSaveAs()],
+      ["clear", () => this.clear()],
       ["open image", () => this.onOpen()],
     ];
     const menu = new lively.contextmenu(this, menuElements)
     menu.openIn(document.body, evt, this)
   }
-
-  livelyMigrate(other) {
-    this.target = other.target
-  }
-
+
   get lastSource() {
     return this._lastSource  
   }
@@ -358,13 +361,11 @@ export default class LivelyImageEditor extends Morph {
   }
 
   livelyMigrate(other) {
-
+    this.target = other.target
     this.migrateCanvas = other.canvas
   }
-
-
-  async livelyExample() {
-
+
+  async livelyExample() {    
     this.loadImage("https://lively-kernel.org/lively4/foo/test.png")
   }
 }

src/components/tools/openai-audio-chat.js

@@ -39,6 +39,13 @@ export default class OpenaiAudioChat extends Morph {
   async initialize() {
     this.windowTitle = "OpenAI Audio Chat";
     this.audioRecorder = new AudioRecorder();
+    this.prompt = [
+        {
+        role: 'system',
+        content: 'Play the role of a helpfull AI chat-bot in a JavaScript, HTML, CSS Web-based development environment.'
+        }
+      ]      
+
     if (!this.conversation) {
       this.conversation = [
         // {
@@ -153,7 +160,7 @@ export default class OpenaiAudioChat extends Morph {
       "n": 1,
       "stream": false,
       "stop": "VANILLA",
-      "messages": this.conversation,
+      "messages": this.prompt.concat(this.conversation),
     }
 
     const requestOptions = {