Update to new backend

SQUASHED: SQUASHED-AUTO-COMMIT-src-components-tools-astro-plot.js-AUTO-COMMIT-src-components-tools-astro-view-example-transformer.py-AUTO-COMMIT-src-components-tools-astro-view.html-AUTO-COMMIT-src-components-tools-astro-view.js-AUTO-COMMIT-src-components-tools-astro-view.js.l4a,

src/components/tools/astro-plot.js

@@ -15,7 +15,7 @@ export default class AstroPlot extends Morph {
         Plotly.deleteTraces(container, existingTraceIndex);
     }
 
-    const { umap_embedding, content, cluster } = feature;
+    const { umap_embedding, plot_content, cluster } = feature;
 
     // Add the new yellow point
     const newTrace = {
@@ -38,7 +38,7 @@ export default class AstroPlot extends Morph {
       },
       customdata: [{
         cluster: 'Cluster ' + cluster, 
-        contentAbbr: `${this.displayContent(content)}...`
+        contentAbbr: `${this.displayContent(plot_content)}...`
       }],
       hovertemplate: `
         <b>%{customdata.cluster}</b><br>
@@ -57,6 +57,10 @@ export default class AstroPlot extends Morph {
   }
 
   displayContent(str) {
+    if (typeof(str) !== 'string') {
+      console.warn('empty string');
+      return '<empty>';
+    }
     return str.slice(0, 100).replace('\n', '<br>');
   }
 
@@ -74,20 +78,23 @@ export default class AstroPlot extends Morph {
 
     features.forEach(({ 
       umap_embedding,
-      function_name,
-      content = "",
+      plot_title,
+      plot_content,
+      file,
       id
     }, i) => dataframe._push({
       x: umap_embedding[0],
       y: umap_embedding[1],
       z: umap_embedding[2],
-      text: function_name,
       color: clusters[i],
       customdata: { 
         cluster: 'Cluster ' + clusters[i], 
-        content, 
-        contentAbbr: `${this.displayContent(content)}...`
+        content: plot_content,
+        title: plot_title,
+        file,
+        contentAbbr: `${this.displayContent(plot_content)}...`
       },
+      // text: ' ' + plot_title, // ' '+ neccessary!!
       ids: id || i
     }));
 
@@ -105,27 +112,47 @@ export default class AstroPlot extends Morph {
           color: clusters,
           colorscale: 'Viridis',
         },
+        selected: {
+          marker: {
+            size: 15,         
+            line: {
+              width: 4
+            }
+          }
+        },
+        unselected: {
+          marker: {
+            size: 8,
+          }
+        },
         hoverlabel: {
           align: 'left'
         },
         hovertemplate: `
 <b>%{customdata.cluster}</b><br>
-%{text}
-<br><br>
-
-<b>Code:</b><br>
+<b>Path</b>
+<br>
+%{customdata.file}
+<br>
+<br>
+<b>Code:</b>
+<br>
 %{customdata.contentAbbr}
 <extra></extra>`,
-        type: 'scatter3d'
+        type: 'scatter3d',
+        selectedpoints: [0, 1]
       }
     ]
 
-    var layout = {margin: {
-      l: 0,
-      r: 0,
-      b: 0,
-      t: 0
-    }};
+    var layout = {
+      margin: {
+        l: 0,
+        r: 0,
+        b: 0,
+        t: 0,
+      },
+      clickmode: 'event+select'
+    };
 
     let container = this.get('#embedding_plot')
     container.innerHTML = "";
@@ -134,6 +161,17 @@ export default class AstroPlot extends Morph {
       responsive: true,
       displayModeBar: false
     });
+
+
+    container.on('plotly_click', (data) => {
+      let item = data.points[0];
+
+      // Update the plot with new selected points
+      // Plotly.restyle(container, 'selectedpoints', [item.pointNumber], [item.curveNumber]);
+
+      this.dispatchEvent(new CustomEvent('item_click', { detail: item }));
+    });
+
   }
 
   async initialize() {

src/components/tools/astro-view-example-transformer.py

@@ -1,41 +1,54 @@
-class WeightedEmbedding(CodeTransformation):
-    def queryAST(self):
-        return '''
-            (class_declaration
-                (type_identifier) @class_name
-                (class_body
-                    (method_definition
-                        name: (property_identifier) @method_name
-                        body: (statement_block) @method_body
-                    ) @method
-                )
-            ) @class
-        '''
+class ConcatIdentifierEmbedding(CodeTransformation):
+    def context(self):
+        return 'file'
+
+    def query(self, AST, line):
+        return AST('''
+          (class_declaration
+              (type_identifier) @class_name
+              (class_body
+                  (method_definition
+                      name: (property_identifier) @method_name
+                      body: (statement_block) @method_body
+                  ) @method
+              )
+          ) @class
+        ''')
 
-    async def mapCaptures(self, query_result, text_embedding, make_query):
-        # (id, path, query_id, captures, _) = query_result
+    def map(self, match, context_embedding, query_node):
+        (id, path, query_id, captures) = match
 
-        [class_name, method_name, method_body] = [
-            self.textFromCapture(query_result, 'class_name'), 
-            self.textFromCapture(query_result, 'method_name'), 
-            self.textFromCapture(query_result, 'method_body')
-        ]
+        # average of all embeddings for tokens in the method body
+        method_node = captures['@method']
+        method_embeddings = context_embedding(method_node)
+        method_embedding = np.mean(method_embeddings, axis=0)
+
+        class_name_embedding = context_embedding(captures['@class_name'])[0]
 
-        [class_embedding, method_name_embedding, method_body_embedding] = await asyncio.gather(
-            text_embedding(class_name),
-            text_embedding(method_name),
-            text_embedding(method_body)
-        )
+        id_matches = query_node('(identifier) @identifier', method_node)
+        if len(id_matches) > 0:
+            identifier_nodes = [match['@identifier'] for match in id_matches]
+
+            identifier_embeddings = np.array([context_embedding(node)[0] for node in identifier_nodes])
+            identifier_mean = np.mean(identifier_embeddings, axis=0)
+        else:
+            identifier_mean = np.zeros(method_embedding.shape[0])
 
         # return dict with embeddings
         return {
-            "class_embedding": np.array(class_embedding), 
-            "method_name_embedding": np.array(method_name_embedding), 
-            "method_body_embedding": np.array(method_body_embedding)
+            "class_name_embedding": class_name_embedding,
+            "method_embedding": method_embedding,
+            "identifier_mean": identifier_mean,
+            "plot_title": captures['@method_name'].text.decode(),
+            "plot_content": captures['@method'].text.decode()
         }
 
     def reduce(self, df):
+        # weighted sum of embeddings
+        # class_embedding 0.1, method_name_embedding 0.2, method_body_embedding 0.7
+        # multiply whole columns by respective scalar, then add them together
+
         return \
-            df['class_embedding'] * 0.2 + \
-            df['method_name_embedding'] * 0.1 + \
-            df['method_body_embedding'] * 0.7
+            df['method_embedding'] * 0.8 + \
+            df['class_name_embedding'] * 0.1 + \
+            df['identifier_mean'] + 0.1

src/components/tools/astro-view.html

@@ -160,7 +160,9 @@
           <div id="status"/>
         </div>
 
-        <lively-editor class="pane editor" id="transformerSource"></lively-editor>
+        <div>
+          <lively-editor class="pane editor" id="transformerSource"></lively-editor>
+        </div>
 
         <lively-separator></lively-separator>
 

src/components/tools/astro-view.js

@@ -21,7 +21,7 @@ export default class AstroView extends Morph {
 
   // Status Text
   get statusLine() { return this.get("#status"); }
-  set status(text) { this.statusLine.innerText = text; }
+  set status(text) { this.statusLine.innerHTML = text; }
 
   // Source
   get sourceEditor() { return this.get("#source"); }  
@@ -149,6 +149,8 @@ export default class AstroView extends Morph {
     this.transformerSourcePath.addEventListener("keyup", evt => {
       if (evt.code == "Enter") this.onTransformerSourcePathEntered(this.transformerSourcePath.value);
     });
+
+    this.astroPlot.addEventListener('item_click', evt => this.onItemClicked(evt));
 
     const source = this.getAttribute("source");
     if (source) this.loadSourceFile(source);
@@ -324,7 +326,7 @@ export default class AstroView extends Morph {
       })
       response = await response.json();
       if (response.error) throw new Error(response.error);
-      this.status = "parser: currently " + response.ASTs + " ASTs in memory ";
+      this.status = `parser: currently ${response.ASTs} ASTs in memory. ${response.features} tokens with embeddings`;
     } catch (e) {
       this.status = "parser: " + e;
     }
@@ -352,7 +354,7 @@ export default class AstroView extends Morph {
       })
       response = await response.json();
       if (response.error) throw new Error(response.error);
-      this.status = "map: success. Data columns: " + response.columns;
+      this.status = ("map: success. Data columns: \n" + response.columns).replaceAll('\n', '<br/>');
     } catch (e) {
       this.status = "map: " + e;
     }
@@ -387,14 +389,36 @@ export default class AstroView extends Morph {
       this.status = "umap: " + e;
       return;
     }
-    debugger;
 
     try {
       this.astroPlot.displayData(data)
     } catch (e) {
       this.status = "plot: " + e;
     }
   }
+
+  async onItemClicked(e) {
+    const item = e.detail;
+    let id = item.id;
+
+    this.status = "get item: running..."
+    try {
+      let response = await fetch(`${this.api}/dataset/${this.projectName}/embeddings/${id}`, {
+        method: 'GET',
+      })
+      response = await response.json();
+      if (response.error) throw new Error(response.error);
+      this.status = "get item: success";
+
+      const item = JSON.parse(response.item);
+      const source = item.plot_content;
+
+      this.sourceCM.setValue(source);
+      debugger
+    } catch (e) {
+      this.status = "get item: " + e;
+    }
+  }
 
   async save() {
     if (this.sourceURL) {