feat(compiler): docstrings (#6428)

Resolves #4446

docstrings can be specified with `///`:
```wing
/// This is a doc string for `Foo`
/// And it has another line to it
class Foo {
  /// This is a docstring for `x`
  x: num;
}
```
![image](https://github.com/winglang/wing/assets/1160578/2131e638-3cd0-4c6b-a6c7-53fa8f5e2f8e)

## more stuff
Added support for hovering over the ctor and the class name in in `new` expressions to display ctor docs (instead of class docs).

## Checklist

- [x] Title matches [Winglang's style guide](https://www.winglang.io/contributing/start-here/pull_requests#how-are-pull-request-titles-formatted)
- [x] Description explains motivation and solution
- [x] Tests added (always)
- [ ] Docs updated (only required for features)
- [ ] Added `pr/e2e-full` label if this feature requires end-to-end testing

*By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*.

examples/tests/valid/class.test.w

@@ -212,4 +212,19 @@ new Baz();
 class Boom {
   new() { }
 }
-class Bam extends Boom {}
+class Bam extends Boom {}
+
+// A documented class (should be parssed without errors)
+/// Class documentation
+/// blah blah blah
+class DocClass {
+  /// Method documentation
+  /// blah blah blah
+  docMethod() {}
+  /// Field documentation
+  /// blah blah blah
+  docField: num;
+  /// Ctor documentation
+  /// blah blah blah
+  new() { this.docField = 0; }
+}

examples/tests/valid/enums.test.w

@@ -31,3 +31,12 @@ test "toStr inflight" {
   assert("{SomeEnum.TWO}" == "TWO");
   assert("{SomeEnum.THREE}" == "THREE");
 }
+
+// A documented enum (should be parssed without errors)
+/// Enum documentation
+/// blah blah blah
+enum DocumentedEnum {
+    /// Variant documentation
+    /// blah blah blah
+    VARIANT
+}

examples/tests/valid/external_ts.extern.d.ts

@@ -187,7 +187,14 @@ export interface IHostedLiftable extends ILiftable {
 }
 /** Abstract interface for `Resource`. */
 export interface IResource extends IConstruct, IHostedLiftable {
+  /** The tree node. */
   readonly node: Node;
+  /** A hook called by the Wing compiler once for each inflight host that needs to use this object inflight.
+  The list of requested inflight methods
+  needed by the inflight host are given by `ops`.
+  
+  This method is commonly used for adding permissions, environment variables, or
+  other capabilities to the inflight host. */
   readonly onLift: (host: IInflightHost, ops: (readonly (string)[])) => void;
 }
 /** Shared behavior between all Wing SDK resources. */
@@ -207,10 +214,22 @@ containers (such as ECS or Kubernetes), VMs or even physical servers.
 
 This data represents the code together with the bindings to preflight data required to run. */
 export interface IInflight extends IHostedLiftable {
+  /** A hook called by the Wing compiler once for each inflight host that needs to use this object inflight.
+  The list of requested inflight methods
+  needed by the inflight host are given by `ops`.
+  
+  This method is commonly used for adding permissions, environment variables, or
+  other capabilities to the inflight host. */
   readonly onLift: (host: IInflightHost, ops: (readonly (string)[])) => void;
 }
 /** A resource with an inflight "handle" method that can be passed to the bucket events. */
 export interface IBucketEventHandler extends IInflight {
+  /** A hook called by the Wing compiler once for each inflight host that needs to use this object inflight.
+  The list of requested inflight methods
+  needed by the inflight host are given by `ops`.
+  
+  This method is commonly used for adding permissions, environment variables, or
+  other capabilities to the inflight host. */
   readonly onLift: (host: IInflightHost, ops: (readonly (string)[])) => void;
 }
 /** `onCreate` event options. */

examples/tests/valid/interface.test.w

@@ -41,3 +41,12 @@ interface IThing1 {
 interface IThing2 {
   m1(): IThing1?;
 }
+
+// A documented interface (should be parsed without errors)
+/// Interface documentation
+/// blah blah blah
+interface IDocumentedInterface {
+    /// Method documentation
+    /// blah blah blah
+    method1(): void;
+}

examples/tests/valid/structs.test.w

@@ -100,4 +100,13 @@ struct Node {
 let aNode = Node {val: "someval"};
 let bNode = Node {val: "otherval", next: aNode};
 
-expect.equal(Json.stringify(bNode), "\{\"val\":\"otherval\",\"next\":\{\"val\":\"someval\"\}\}");
+expect.equal(Json.stringify(bNode), "\{\"val\":\"otherval\",\"next\":\{\"val\":\"someval\"\}\}");
+
+// A documented struct (should be parsed without errors)
+/// Struct documentation
+/// blah blah blah
+struct DocumentedStruct {
+    /// Field documentation
+    /// blah blah blah
+    field: str;
+}

libs/tree-sitter-wing/grammar.js

@@ -22,7 +22,7 @@ const PREC = {
 module.exports = grammar({
   name: "wing",
 
-  extras: ($) => [$.comment, /[\s\p{Zs}\uFEFF\u2060\u200B]/],
+  extras: ($) => [$.comment, $.doc, /[\s\p{Zs}\uFEFF\u2060\u200B]/],
 
   word: ($) => $.identifier,
 
@@ -54,9 +54,10 @@ module.exports = grammar({
       choice(braced(optional(repeat($._statement))), $.AUTOMATIC_BLOCK),
     _semicolon: ($) => choice(";", $.AUTOMATIC_SEMICOLON),
     comment: ($) =>
-      token(
-        choice(seq("//", /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/"))
-      ),
+      token(choice(seq(/\/\/[^\/]/, /.*/), seq("/*", /[^*]*\*+([^/*][^*]*\*+)*/, "/"))),
+
+    doc: ($) => seq("///", field("content", $.doc_content)),
+    doc_content: ($) => /.*/,
 
     // Identifiers
     reference: ($) =>
@@ -504,7 +505,7 @@ module.exports = grammar({
     initializer: ($) =>
       seq(
         optional(field("inflight", $.inflight_specifier)),
-        "new",
+        field("ctor_name", "new"),
         field("parameter_list", $.parameter_list),
         field("block", $.block)
       ),

libs/tree-sitter-wing/test/corpus/comments.txt

@@ -0,0 +1,22 @@
+================================================================================
+Docstrings & comments
+================================================================================
+
+// This is a plain comment
+/// This is a doc string
+/// This is another one
+// This too is a plain comment
+
+/// This is a doc after an empty line
+
+--------------------------------------------------------------------------------
+
+(source
+  (comment)
+  (doc
+    content: (doc_content))
+  (doc
+    content: (doc_content))
+  (comment)
+  (doc
+    content: (doc_content)))

libs/wingc/src/ast.rs

@@ -3,7 +3,7 @@ use std::hash::{Hash, Hasher};
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 use camino::Utf8PathBuf;
-use indexmap::{Equivalent, IndexMap, IndexSet};
+use indexmap::{Equivalent, IndexMap};
 use itertools::Itertools;
 
 use crate::diagnostic::WingSpan;
@@ -295,6 +295,8 @@ pub struct FunctionDefinition {
 	pub is_static: bool,
 	/// Function's access modifier. In case of a closure, this is always public.
 	pub access: AccessModifier,
+	/// Function's documentation
+	pub doc: Option<String>,
 	pub span: WingSpan,
 }
 
@@ -303,6 +305,7 @@ pub struct Stmt {
 	pub kind: StmtKind,
 	pub span: WingSpan,
 	pub idx: usize,
+	pub doc: Option<String>,
 }
 
 #[derive(Debug)]
@@ -391,7 +394,8 @@ impl Class {
 #[derive(Debug)]
 pub struct Interface {
 	pub name: Symbol,
-	pub methods: Vec<(Symbol, FunctionSignature)>,
+	// Each method has a symbol, a signature, and an optional documentation string
+	pub methods: Vec<(Symbol, FunctionSignature, Option<String>)>,
 	pub extends: Vec<UserDefinedType>,
 	pub access: AccessModifier,
 	pub phase: Phase,
@@ -408,7 +412,8 @@ pub struct Struct {
 #[derive(Debug)]
 pub struct Enum {
 	pub name: Symbol,
-	pub values: IndexSet<Symbol>,
+	// Each value has a symbol and an optional documenation string
+	pub values: IndexMap<Symbol, Option<String>>,
 	pub access: AccessModifier,
 }
 
@@ -517,6 +522,7 @@ pub struct ClassField {
 	pub phase: Phase,
 	pub is_static: bool,
 	pub access: AccessModifier,
+	pub doc: Option<String>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq)]
@@ -540,6 +546,7 @@ impl Display for AccessModifier {
 pub struct StructField {
 	pub name: Symbol,
 	pub member_type: TypeAnnotation,
+	pub doc: Option<String>,
 }
 
 #[derive(Debug)]

libs/wingc/src/closure_transform.rs

@@ -181,6 +181,7 @@ impl Fold for ClosureTransformer {
 					// we need to set this to false.
 					is_static: false,
 					access: AccessModifier::Public,
+					doc: None,
 				};
 
 				// class_init_body :=
@@ -220,6 +221,7 @@ impl Fold for ClosureTransformer {
 						value: Expr::new(ExprKind::Literal(Literal::Boolean(true)), WingSpan::for_file(file_id)),
 					},
 					span: WingSpan::for_file(file_id),
+					doc: None,
 				}];
 
 				// If we are inside a scope with "this", add define `let __parent_this_${CLOSURE_COUNT} = this` which can be
@@ -242,6 +244,7 @@ impl Fold for ClosureTransformer {
 						},
 						span: WingSpan::for_file(file_id),
 						idx: 0,
+						doc: None,
 					};
 					self.class_statements.push(parent_this_def);
 				}
@@ -271,6 +274,7 @@ impl Fold for ClosureTransformer {
 							body: FunctionBody::Statements(Scope::new(class_init_body, WingSpan::for_file(file_id))),
 							span: WingSpan::for_file(file_id),
 							access: AccessModifier::Public,
+							doc: None,
 						},
 						fields: class_fields,
 						implements: vec![],
@@ -290,12 +294,14 @@ impl Fold for ClosureTransformer {
 							body: FunctionBody::Statements(Scope::new(vec![], WingSpan::for_file(file_id))),
 							span: WingSpan::for_file(file_id),
 							access: AccessModifier::Public,
+							doc: None,
 						},
 						access: AccessModifier::Private,
 						auto_id: true,
 					}),
 					idx: self.nearest_stmt_idx,
 					span: WingSpan::for_file(file_id),
+					doc: None,
 				};
 
 				// new_class_instance :=

libs/wingc/src/docs.rs

@@ -382,7 +382,12 @@ fn render_enum(e: &Enum) -> String {
 	}
 
 	for prop in e.values.iter() {
-		markdown.line(&format!("- `{}`\n", prop));
+		let value_doc = if let Some(doc) = prop.1.as_ref() {
+			format!(" — {}", doc)
+		} else {
+			String::default()
+		};
+		markdown.line(&format!("- `{}{}`\n", prop.0, value_doc));
 	}
 
 	markdown.empty_line();

libs/wingc/src/dtsify/extern_dtsify.rs

@@ -198,7 +198,7 @@ impl<'a> ExternDTSifier<'a> {
 		code.open(format!("export enum {} {{", enum_.name.name));
 
 		for (i, variant) in enum_.values.iter().enumerate() {
-			code.line(format!("{variant} = {i},"));
+			code.line(format!("{} = {i},", variant.0));
 		}
 
 		code.close("}");

libs/wingc/src/dtsify/mod.rs

@@ -280,7 +280,7 @@ impl<'a> DTSifier<'a> {
 			StmtKind::Enum(enu) => {
 				code.open(format!("export enum {} {{", enu.name.name));
 				for value in &enu.values {
-					code.line(format!("{},", value.name));
+					code.line(format!("{},", value.0.name));
 				}
 				code.close("}");
 			}