Add whitespace escape (kdl v2)

CHANGES.md

@@ -5,6 +5,8 @@
 - Updated to KDL v2.0:
 - - Added Line Tabulation U+000B to whitespace characters
 - - Removed `\/` from escape sequences
+- - Added whitespace escape sequence: all whitespace (including multiple
+    newlines) is removed after `\`
 - - Identifiers cannot start with `r#` anymore
 - Dropped support for OCaml < 4.10.0
 

src/lexer.ml

@@ -105,6 +105,12 @@ let rec line_comment lexbuf =
   | any -> line_comment lexbuf
   | _ -> assert false
 
+let rec whitespace_escape lexbuf =
+  match%sedlex lexbuf with
+  | newline -> new_line lexbuf; whitespace_escape lexbuf
+  | ws -> whitespace_escape lexbuf
+  | _ -> ()
+
 let rec raw_string hashlen lexbuf =
   match%sedlex lexbuf with
   | newline ->
@@ -141,6 +147,7 @@ let rec string lexbuf =
   | "\\b" -> Buffer.add_char string_buffer '\b'; string lexbuf
   | "\\f" -> Buffer.add_char string_buffer '\012'; string lexbuf
   | "\\u{", Rep (hex_digit, 1 .. 6), '}' ->
+    (* TODO: Disallow 7+ hex digits inside \u{...}? *)
     let code_str =
       Sedlexing.Utf8.sub_lexeme lexbuf 3 (Sedlexing.lexeme_length lexbuf - 4) in
     let code = int_of_string @@ "0x" ^ code_str in
@@ -150,6 +157,9 @@ let rec string lexbuf =
       error "Invalid unicode code point";
     Buffer.add_utf_8_uchar string_buffer (Uchar.unsafe_of_int code);
     string lexbuf
+  | '\\', ws -> whitespace_escape lexbuf; string lexbuf
+  | '\\', newline -> new_line lexbuf; whitespace_escape lexbuf; string lexbuf
+  (* TODO: Should we error in case of invalid escape sequences? *)
   | '"' ->
     STRING (Buffer.contents string_buffer)
   | eof -> error "Unterminated string"

test/parse.ml

@@ -389,6 +389,28 @@ let%test_module "strings" = (module struct
   let%expect_test "empty \\u{} without the code point" =
     test {|- "\u{}"|};
     [%expect {| (- (string "\\u{}")) |}]
+
+  let%expect_test "whitespace escape should remove space from the string" =
+    test {|- "foo \   bar"|};
+    [%expect {| (- (string "foo bar")) |}]
+
+  let%expect_test "whitespace escape should also remove a newline" =
+    test {|- "foo \
+              bar"|};
+    [%expect {| (- (string "foo bar")) |}]
+
+  let%expect_test "whitespace escape should remove multiple newlines" =
+    (* This behavior is the same as in Rust
+       https://github.com/rust-lang/reference/pull/1042 *)
+    test {|- "foo\
+
+
+            bar"|};
+    [%expect {| (- (string foobar)) |}]
+
+  let%expect_test "whitespace escape preceding space and then newline" =
+    test "- \"foo \\ \n bar\"";
+    [%expect {| (- (string "foo bar")) |}]
 end)
 
 let%test_module "line continuations" = (module struct