Merge pull request #45 from ocaml-wasm/target-dir

Always put Wasm code and source maps into an asset directory

.github/workflows/build-wasm_of_ocaml.yml

@@ -123,7 +123,7 @@ jobs:
 
       - name: Pin dune
         run: |
-          opam pin add -n dune.3.13 https://github.com/ocaml-wasm/dune.git#wasm-separate-compilation
+          opam pin add -n dune.3.17 https://github.com/ocaml-wasm/dune.git#wasm_of_ocaml
 
       - name: Pin wasm_of_ocaml
         working-directory: ./wasm_of_ocaml
@@ -137,7 +137,7 @@ jobs:
         run: opam exec -- ocaml wasm_of_ocaml/tools/ci_setup.ml
 
       - name: Update test dependencies
-        run: opam install num cohttp-lwt-unix ppx_expect
+        run: opam install num cohttp-lwt-unix ppx_expect cstruct
 
       - name: Install wasm_of_ocaml
         working-directory: ./wasm_of_ocaml

README.md

@@ -28,7 +28,7 @@ The following commands will perform a minimal installation:
 ```
 git clone https://github.com/ocaml-wasm/wasm_of_ocaml
 cd wasm_of_ocaml
-opam pin add dune.3.13 https://github.com/ocaml-wasm/dune.git#wasm
+opam pin add dune.3.17 https://github.com/ocaml-wasm/dune.git#wasm_of_ocaml
 opam pin add -n --with-version 5.3.0-wasm .
 opam install wasm_of_ocaml-compiler
 ```

compiler/bin-wasm_of_ocaml/compile.ml

@@ -67,7 +67,6 @@ let link_and_optimize
     ~sourcemap_root
     ~sourcemap_don't_inline_content
     ~opt_sourcemap
-    ~opt_sourcemap_url
     runtime_wasm_files
     wat_files
     output_file =
@@ -113,7 +112,6 @@ let link_and_optimize
     ~profile
     ~opt_input_sourcemap:opt_temp_sourcemap'
     ~opt_output_sourcemap:opt_sourcemap
-    ~opt_sourcemap_url
     ~input_file:temp_file'
     ~output_file;
   Option.iter
@@ -136,7 +134,6 @@ let link_runtime ~profile runtime_wasm_files output_file =
     ~profile
     ~opt_input_sourcemap:None
     ~opt_output_sourcemap:None
-    ~opt_sourcemap_url:None
     ~input_file:temp_file
     ~output_file
 
@@ -169,8 +166,7 @@ let build_prelude z =
     ~input_file:prelude_file
     ~output_file:tmp_prelude_file
     ~opt_input_sourcemap:None
-    ~opt_output_sourcemap:None
-    ~opt_sourcemap_url:None;
+    ~opt_output_sourcemap:None;
   Zip.add_file z ~name:"prelude.wasm" ~file:tmp_prelude_file;
   predefined_exceptions
 
@@ -365,8 +361,6 @@ let run
          ~profile
          ~opt_input_sourcemap:None
          ~opt_output_sourcemap
-         ~opt_sourcemap_url:
-           (if enable_source_maps then Some (unit_name ^ ".wasm.map") else None)
          ~input_file:wat_file
          ~output_file:tmp_wasm_file;
        Option.iter
@@ -393,35 +387,43 @@ let run
          if times () then Format.eprintf "  parsing: %a@." Timer.print t1;
          Fs.gen_file (Filename.chop_extension output_file ^ ".wat")
          @@ fun wat_file ->
-         let wasm_file =
-           if Filename.check_suffix output_file ".wasm.js"
-           then Filename.chop_extension output_file
-           else Filename.chop_extension output_file ^ ".wasm"
+         let dir = Filename.chop_extension output_file ^ ".assets" in
+         Fs.gen_file dir
+         @@ fun tmp_dir ->
+         Sys.mkdir tmp_dir 0o777;
+         let opt_sourcemap =
+           if enable_source_maps
+           then Some (Filename.concat tmp_dir "code.wasm.map")
+           else None
          in
-         Fs.gen_file wasm_file
-         @@ fun tmp_wasm_file ->
-         opt_with
-           Fs.gen_file
-           (if enable_source_maps then Some (wasm_file ^ ".map") else None)
-         @@ fun opt_tmp_sourcemap ->
          let generated_js = output_gen wat_file (output code ~unit_name:None) in
+         let tmp_wasm_file = Filename.concat tmp_dir "code.wasm" in
          let primitives =
            link_and_optimize
              ~profile
              ~sourcemap_root
              ~sourcemap_don't_inline_content
-             ~opt_sourcemap:opt_tmp_sourcemap
-             ~opt_sourcemap_url:
-               (if enable_source_maps
-                then Some (Filename.basename wasm_file ^ ".map")
-                else None)
+             ~opt_sourcemap
              runtime_wasm_files
              [ wat_file ]
              tmp_wasm_file
          in
+         let wasm_name =
+           Printf.sprintf
+             "code-%s"
+             (String.sub (Digest.to_hex (Digest.file tmp_wasm_file)) ~pos:0 ~len:20)
+         in
+         let tmp_wasm_file' = Filename.concat tmp_dir (wasm_name ^ ".wasm") in
+         Sys.rename tmp_wasm_file tmp_wasm_file';
+         if enable_source_maps
+         then (
+           Sys.rename (Filename.concat tmp_dir "code.wasm.map") (tmp_wasm_file' ^ ".map");
+           Wa_link.Wasm_binary.append_source_map_section
+             ~file:tmp_wasm_file'
+             ~url:(wasm_name ^ ".wasm.map"));
          let js_runtime =
            let missing_primitives =
-             let l = Wa_link.Wasm_binary.read_imports ~file:tmp_wasm_file in
+             let l = Wa_link.Wasm_binary.read_imports ~file:tmp_wasm_file' in
              List.filter_map
                ~f:(fun { Wa_link.Wasm_binary.module_; name; _ } ->
                  if String.equal module_ "env" then Some name else None)
@@ -432,7 +434,9 @@ let run
              ~runtime_arguments:
                (Wa_link.build_runtime_arguments
                   ~missing_primitives
-                  ~wasm_file
+                  ~wasm_dir:dir
+                  ~link_spec:[ wasm_name, None ]
+                  ~separate_compilation:false
                   ~generated_js:[ None, generated_js ]
                   ())
              ()

compiler/lib/wasm/wa_binaryen.ml

@@ -114,13 +114,8 @@ let optimization_options =
    ; [ "-O3"; "--skip-pass=inlining-optimizing"; "--traps-never-happen" ]
   |]
 
-let optimize
-    ~profile
-    ~opt_input_sourcemap
-    ~input_file
-    ~opt_output_sourcemap
-    ~opt_sourcemap_url
-    ~output_file =
+let optimize ~profile ~opt_input_sourcemap ~input_file ~opt_output_sourcemap ~output_file
+    =
   let level =
     match profile with
     | None -> 1
@@ -132,5 +127,4 @@ let optimize
         @ optimization_options.(level - 1)
         @ [ Filename.quote input_file; "-o"; Filename.quote output_file ])
     @ opt_flag "--input-source-map" opt_input_sourcemap
-    @ opt_flag "--output-source-map" opt_output_sourcemap
-    @ opt_flag "--output-source-map-url" opt_sourcemap_url)
+    @ opt_flag "--output-source-map" opt_output_sourcemap)

compiler/lib/wasm/wa_binaryen.mli

@@ -36,6 +36,5 @@ val optimize :
   -> opt_input_sourcemap:string option
   -> input_file:string
   -> opt_output_sourcemap:string option
-  -> opt_sourcemap_url:string option
   -> output_file:string
   -> unit

compiler/lib/wasm/wa_link.ml

@@ -259,6 +259,30 @@ module Wasm_binary = struct
             find_sections i)
     in
     find_sections { imports = []; exports = [] }
+
+  let append_source_map_section ~file ~url =
+    let ch = open_out_gen [ Open_wronly; Open_append; Open_binary ] 0o666 file in
+    let rec output_uint buf i =
+      if i < 128
+      then Buffer.add_char buf (Char.chr i)
+      else (
+        Buffer.add_char buf (Char.chr (128 + (i land 127)));
+        output_uint buf (i lsr 7))
+    in
+    let buf = Buffer.create 16 in
+    let output_name buf s =
+      output_uint buf (String.length s);
+      Buffer.add_string buf s
+    in
+    output_name buf "sourceMappingURL";
+    output_name buf url;
+    let section_contents = Buffer.contents buf in
+    Buffer.clear buf;
+    Buffer.add_char buf '\000';
+    output_uint buf (String.length section_contents);
+    output_string ch (Buffer.contents buf);
+    output_string ch section_contents;
+    close_out ch
 end
 
 let trim_semi s =
@@ -371,7 +395,6 @@ let generate_start_function ~to_link ~out_file =
     ~profile:(Driver.profile 1)
     ~opt_input_sourcemap:None
     ~opt_output_sourcemap:None
-    ~opt_sourcemap_url:None
     ~input_file:wat_file
     ~output_file:wasm_file;
   if times () then Format.eprintf "    generate start: %a@." Timer.print t1
@@ -411,10 +434,10 @@ let report_missing_primitives missing =
     List.iter ~f:(fun nm -> warn "  %s@." nm) missing)
 
 let build_runtime_arguments
-    ?(link_spec = [])
-    ?(separate_compilation = false)
+    ~link_spec
+    ~separate_compilation
     ~missing_primitives
-    ~wasm_file
+    ~wasm_dir
     ~generated_js
     () =
   let missing_primitives = if Config.Flag.genprim () then missing_primitives else [] in
@@ -520,29 +543,26 @@ let build_runtime_arguments
   in
   obj
     [ ( "link"
-      , if List.is_empty link_spec
-        then ENum (Javascript.Num.of_int32 (if separate_compilation then 1l else 0l))
-        else
-          EArr
-            (List.map
-               ~f:(fun (m, deps) ->
-                 Javascript.Element
-                   (EArr
-                      [ Element (EStr (Utf8_string.of_string_exn m))
-                      ; Element
-                          (match deps with
-                          | None -> ENum (Javascript.Num.of_int32 0l)
-                          | Some l ->
-                              EArr
-                                (List.map
-                                   ~f:(fun i ->
-                                     Javascript.Element
-                                       (ENum (Javascript.Num.of_int32 (Int32.of_int i))))
-                                   l))
-                      ]))
-               link_spec) )
+      , EArr
+          (List.map
+             ~f:(fun (m, deps) ->
+               Javascript.Element
+                 (EArr
+                    [ Element (EStr (Utf8_string.of_string_exn m))
+                    ; Element
+                        (match deps with
+                        | None -> ENum (Javascript.Num.of_int32 0l)
+                        | Some l ->
+                            EArr
+                              (List.map
+                                 ~f:(fun i ->
+                                   Javascript.Element
+                                     (ENum (Javascript.Num.of_int32 (Int32.of_int i))))
+                                 l))
+                    ]))
+             link_spec) )
     ; "generated", generated_js
-    ; "src", EStr (Utf8_string.of_string_exn (Filename.basename wasm_file))
+    ; "src", EStr (Utf8_string.of_string_exn (Filename.basename wasm_dir))
     ]
 
 let link_to_directory ~set_to_link ~files ~enable_source_maps ~dir =
@@ -742,7 +762,7 @@ let link ~output_file ~linkall ~enable_source_maps ~files =
   if times () then Format.eprintf "    finding what to link: %a@." Timer.print t1;
   if times () then Format.eprintf "  scan: %a@." Timer.print t;
   let t = Timer.make () in
-  let interfaces, wasm_file, link_spec =
+  let interfaces, wasm_dir, link_spec =
     let dir = Filename.chop_extension output_file ^ ".assets" in
     Fs.gen_file dir
     @@ fun tmp_dir ->
@@ -786,7 +806,7 @@ let link ~output_file ~linkall ~enable_source_maps ~files =
         ~link_spec
         ~separate_compilation:true
         ~missing_primitives
-        ~wasm_file
+        ~wasm_dir
         ~generated_js
         ()
     in

compiler/lib/wasm/wa_link.mli

@@ -25,6 +25,8 @@ module Wasm_binary : sig
     }
 
   val read_imports : file:string -> import list
+
+  val append_source_map_section : file:string -> url:string -> unit
 end
 
 type unit_data =
@@ -42,10 +44,10 @@ val add_info :
   -> unit
 
 val build_runtime_arguments :
-     ?link_spec:(string * int list option) list
-  -> ?separate_compilation:bool
+     link_spec:(string * int list option) list
+  -> separate_compilation:bool
   -> missing_primitives:string list
-  -> wasm_file:string
+  -> wasm_dir:string
   -> generated_js:
        (string option * (string list * (string * Javascript.expression) list)) list
   -> unit

dune

@@ -14,15 +14,15 @@
    (tools/node_wrapper.sh as node))
   (js_of_ocaml
    (compilation_mode separate)
-   (targets wasm)))
+   (submodes wasm)))
  (wasm-effects
   (binaries
    (tools/node_wrapper.sh as node))
   (js_of_ocaml
    (compilation_mode separate)
    (flags
     (:standard --enable effects))
-   (targets wasm)))
+   (submodes wasm)))
  (bench_no_debug
   (flags
    (:standard \ -g))

dune-project

@@ -1,4 +1,4 @@
-(lang dune 3.11)
+(lang dune 3.17)
 (using menhir 2.0)
 (name js_of_ocaml)
 

js_of_ocaml-compiler.opam

@@ -12,7 +12,7 @@ homepage: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 doc: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 bug-reports: "https://github.com/ocsigen/js_of_ocaml/issues"
 depends: [
-  "dune" {>= "3.11"}
+  "dune" {>= "3.17"}
   "ocaml" {>= "4.08" & < "5.1"}
   "num" {with-test}
   "ppx_expect" {>= "v0.14.2" & with-test}

js_of_ocaml-lwt.opam

@@ -12,7 +12,7 @@ homepage: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 doc: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 bug-reports: "https://github.com/ocsigen/js_of_ocaml/issues"
 depends: [
-  "dune" {>= "3.11"}
+  "dune" {>= "3.17"}
   "ocaml" {>= "4.08"}
   "js_of_ocaml" {= version}
   "js_of_ocaml-ppx" {= version}

js_of_ocaml-ppx.opam

@@ -12,7 +12,7 @@ homepage: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 doc: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 bug-reports: "https://github.com/ocsigen/js_of_ocaml/issues"
 depends: [
-  "dune" {>= "3.11"}
+  "dune" {>= "3.17"}
   "ocaml" {>= "4.08"}
   "js_of_ocaml" {= version}
   "ppxlib" {>= "0.15.0"}

js_of_ocaml-ppx_deriving_json.opam

@@ -12,7 +12,7 @@ homepage: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 doc: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 bug-reports: "https://github.com/ocsigen/js_of_ocaml/issues"
 depends: [
-  "dune" {>= "3.11"}
+  "dune" {>= "3.17"}
   "ocaml" {>= "4.08"}
   "js_of_ocaml" {= version}
   "ppxlib" {>= "0.15"}

js_of_ocaml-toplevel.opam

@@ -12,7 +12,7 @@ homepage: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 doc: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 bug-reports: "https://github.com/ocsigen/js_of_ocaml/issues"
 depends: [
-  "dune" {>= "3.11"}
+  "dune" {>= "3.17"}
   "ocaml" {>= "4.08"}
   "js_of_ocaml-compiler" {= version}
   "ocamlfind" {>= "1.5.1"}

js_of_ocaml-tyxml.opam

@@ -12,7 +12,7 @@ homepage: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 doc: "https://ocsigen.org/js_of_ocaml/latest/manual/overview"
 bug-reports: "https://github.com/ocsigen/js_of_ocaml/issues"
 depends: [
-  "dune" {>= "3.11"}
+  "dune" {>= "3.17"}
   "ocaml" {>= "4.08"}
   "js_of_ocaml" {= version}
   "js_of_ocaml-ppx" {= version}