Reduce interleaving in the scheduler (#586)

CHANGES.md

@@ -6,6 +6,9 @@
 - Always produce an implementation for the alias module, for
   non-jbuilder users (Fix #576)
 
+- Reduce interleaving in the scheduler in an attempt to make Jbuilder
+  keep file descriptors open for less long (#586)
+
 - Accept and ignore upcoming new library fields: `ppx.driver`,
   `inline_tests` and `inline_tests.backend`
 

src/scheduler.ml

@@ -110,6 +110,16 @@ let wait_for_process pid =
   Running_jobs.add { pid; ivar };
   Fiber.Ivar.read ivar
 
+let rec restart_waiting_for_available_job t =
+  if Queue.is_empty t.waiting_for_available_job ||
+     Running_jobs.count () >= t.concurrency then
+    Fiber.return ()
+  else begin
+    Fiber.Ivar.fill (Queue.pop t.waiting_for_available_job) t
+    >>= fun () ->
+    restart_waiting_for_available_job t
+  end
+
 let rec go_rec t =
   Fiber.yield ()
   >>= fun () ->
@@ -134,13 +144,10 @@ let rec go_rec t =
         t.status_line <- status_line;
     end;
     let job, status = Running_jobs.wait () in
-    (if not (Queue.is_empty t.waiting_for_available_job) then
-       Fiber.Ivar.fill (Queue.pop t.waiting_for_available_job) t
-     else
-       Fiber.return ())
-    >>= fun () ->
     Fiber.Ivar.fill job.ivar status
     >>= fun () ->
+    restart_waiting_for_available_job t
+    >>= fun () ->
     go_rec t
   end
 

test/blackbox-tests/test-cases/byte-code-only/run.t

@@ -1,7 +1,7 @@
   $ env ORIG_PATH="$PATH" PATH="$PWD/ocaml-bin:$PATH" $JBUILDER build --root . -j1 --display short
       ocamldep bin/toto.ml.d
-      ocamldep src/foo.ml.d
         ocamlc bin/.toto.eobjs/toto.{cmi,cmo,cmt}
-        ocamlc src/.foo.objs/foo.{cmi,cmo,cmt}
         ocamlc bin/toto.exe
+      ocamldep src/foo.ml.d
+        ocamlc src/.foo.objs/foo.{cmi,cmo,cmt}
         ocamlc src/foo.cma

test/blackbox-tests/test-cases/c-stubs/run.t

@@ -1,9 +1,9 @@
   $ $JBUILDER exec -j1 ./qnativerun/run.exe --display short --root .
       ocamldep qnativerun/run.ml.d
         ocamlc q/q_stub.o
+    ocamlmklib q/dllq_stubs.so,q/libq_stubs.a
       ocamldep q/q.ml.d
       ocamldep q/q.mli.d
-    ocamlmklib q/dllq_stubs.so,q/libq_stubs.a
         ocamlc q/.q.objs/q.{cmi,cmti}
         ocamlc qnativerun/.run.eobjs/run.{cmi,cmo,cmt}
       ocamlopt q/.q.objs/q.{cmx,o}

test/blackbox-tests/test-cases/copy_files/run.t

@@ -1,17 +1,17 @@
   $ $JBUILDER build -j1 test.exe .merlin --display short --root . --debug-dependency-path
       ocamllex lexers/lexer1.ml
-      ocamldep test.ml.d
       ocamldep lexer1.ml.d
+      ocamldep test.ml.d
       ocamldep dummy.ml.d
-        ocamlc bar.o
         ocamlc .foo.objs/dummy.{cmi,cmo,cmt}
-    ocamlmklib dllfoo_stubs.so,libfoo_stubs.a
         ocamlc .test.eobjs/lexer1.{cmi,cmo,cmt}
-      ocamlopt .foo.objs/dummy.{cmx,o}
         ocamlc .test.eobjs/test.{cmi,cmo,cmt}
+        ocamlc bar.o
+    ocamlmklib dllfoo_stubs.so,libfoo_stubs.a
+      ocamlopt .foo.objs/dummy.{cmx,o}
       ocamlopt .test.eobjs/lexer1.{cmx,o}
-      ocamlopt foo.{a,cmxa}
       ocamlopt .test.eobjs/test.{cmx,o}
+      ocamlopt foo.{a,cmxa}
       ocamlopt test.exe
   $ $JBUILDER build -j1 @bar-source --display short --root .
   #line 1 "include/bar.h"

test/blackbox-tests/test-cases/cross-compilation/run.t

@@ -1,19 +1,19 @@
   $ env OCAMLFIND_CONF=$PWD/etc/findlib.conf $JBUILDER build --display short --root . -j1 -x foo file @install
       ocamldep bin/blah.ml.d [default.foo]
       ocamldep lib/p.ml.d [default.foo]
-      ocamldep bin/blah.ml.d
         ocamlc lib/.p.objs/p.{cmi,cmo,cmt} [default.foo]
-      ocamldep lib/p.ml.d
       ocamlopt lib/.p.objs/p.{cmx,o} [default.foo]
-        ocamlc bin/.blah.eobjs/blah.{cmi,cmo,cmt} [default.foo]
-        ocamlc lib/p.cma [default.foo]
-        ocamlc lib/.p.objs/p.{cmi,cmo,cmt}
       ocamlopt lib/p.{a,cmxa} [default.foo]
-      ocamlopt bin/.blah.eobjs/blah.{cmx,o} [default.foo]
-        ocamlc bin/.blah.eobjs/blah.{cmi,cmo,cmt}
-      ocamlopt lib/.p.objs/p.{cmx,o}
       ocamlopt lib/p.cmxs [default.foo]
+      ocamldep bin/blah.ml.d
+      ocamldep lib/p.ml.d
+        ocamlc lib/.p.objs/p.{cmi,cmo,cmt}
+        ocamlc bin/.blah.eobjs/blah.{cmi,cmo,cmt}
+        ocamlc bin/.blah.eobjs/blah.{cmi,cmo,cmt} [default.foo]
+      ocamlopt bin/.blah.eobjs/blah.{cmx,o} [default.foo]
       ocamlopt bin/blah.exe [default.foo]
+        ocamlc lib/p.cma [default.foo]
+      ocamlopt lib/.p.objs/p.{cmx,o}
       ocamlopt bin/.blah.eobjs/blah.{cmx,o}
       ocamlopt lib/p.{a,cmxa}
       ocamlopt bin/blah.exe

test/blackbox-tests/test-cases/gen-opam-install-file/run.t

@@ -1,18 +1,18 @@
   $ $JBUILDER runtest -j1 --display short --root .
       ocamldep bar.ml.d
       ocamldep foo_byte.ml.d
+        ocamlc .foo_byte.objs/foo_byte.{cmi,cmo,cmt}
+        ocamlc foo_byte.cma
       ocamldep foo.ml.d
       ocamldep foo.mli.d
-        ocamlc .foo_byte.objs/foo_byte.{cmi,cmo,cmt}
         ocamlc .foo.objs/foo.{cmi,cmti}
-        ocamlc foo_byte.cma
         ocamlc .foo.objs/foo.{cmo,cmt}
-      ocamlopt .foo.objs/foo.{cmx,o}
-        ocamlc .bar.eobjs/bar.{cmi,cmo,cmt}
         ocamlc foo.cma
+      ocamlopt .foo.objs/foo.{cmx,o}
       ocamlopt foo.{a,cmxa}
-      ocamlopt .bar.eobjs/bar.{cmx,o}
       ocamlopt foo.cmxs
+        ocamlc .bar.eobjs/bar.{cmi,cmo,cmt}
+      ocamlopt .bar.eobjs/bar.{cmx,o}
       ocamlopt bar.exe
   lib: [
     "_build/install/default/lib/foo/META" {"META"}

test/blackbox-tests/test-cases/github25/root/run.t

@@ -9,9 +9,9 @@ We need ocamlfind to run this test
   $ $JBUILDER build -j1 @install --display short --root . --only hello
         ocamlc .hello.objs/hello.{cmi,cmo,cmt}
       ocamlopt .hello.objs/hello.{cmx,o}
-        ocamlc hello.cma
       ocamlopt hello.{a,cmxa}
       ocamlopt hello.cmxs
+        ocamlc hello.cma
 
   $ $JBUILDER build -j1 @install --display short --root . --only pas-de-bol 2>&1 | sed 's/" in .*/" in .../'
   File "jbuild", line 8, characters 1-107:

test/blackbox-tests/test-cases/installable-dup-private-libs/run.t

@@ -1,13 +1,13 @@
   $ $JBUILDER build @install -j1 --display short --root .
       ocamldep a1/a.ml.d
-      ocamldep a2/a.ml.d
         ocamlc a1/.a.objs/a.{cmi,cmo,cmt}
-        ocamlc a2/.a.objs/a.{cmi,cmo,cmt}
       ocamlopt a1/.a.objs/a.{cmx,o}
-        ocamlc a1/a.cma
-      ocamlopt a2/.a.objs/a.{cmx,o}
-        ocamlc a2/a.cma
       ocamlopt a1/a.{a,cmxa}
-      ocamlopt a2/a.{a,cmxa}
       ocamlopt a1/a.cmxs
+      ocamldep a2/a.ml.d
+        ocamlc a2/.a.objs/a.{cmi,cmo,cmt}
+      ocamlopt a2/.a.objs/a.{cmx,o}
+      ocamlopt a2/a.{a,cmxa}
       ocamlopt a2/a.cmxs
+        ocamlc a1/a.cma
+        ocamlc a2/a.cma

test/blackbox-tests/test-cases/intf-only/run.t

@@ -4,19 +4,19 @@ Successes:
       ocamldep test/bar.ml.d
       ocamldep foo.ml.d
         ocamlc .foo.objs/foo__.{cmi,cmo,cmt}
-      ocamldep intf.mli.d
       ocamlopt .foo.objs/foo__.{cmx,o}
+      ocamldep intf.mli.d
         ocamlc .foo.objs/foo__Intf.{cmi,cmti}
         ocamlc .foo.objs/foo.{cmi,cmo,cmt}
       ocamlopt .foo.objs/foo.{cmx,o}
-        ocamlc test/.bar.objs/bar.{cmi,cmo,cmt}
-        ocamlc foo.cma
       ocamlopt foo.{a,cmxa}
-      ocamlopt test/.bar.objs/bar.{cmx,o}
-        ocamlc test/bar.cma
       ocamlopt foo.cmxs
+        ocamlc test/.bar.objs/bar.{cmi,cmo,cmt}
+      ocamlopt test/.bar.objs/bar.{cmx,o}
       ocamlopt test/bar.{a,cmxa}
       ocamlopt test/bar.cmxs
+        ocamlc foo.cma
+        ocamlc test/bar.cma
 
 Errors:
 

test/blackbox-tests/test-cases/js_of_ocaml/run.t

@@ -1,32 +1,32 @@
   $ $JBUILDER build -j1 --display short --root . --dev bin/technologic.bc.js @install lib/x.cma.js lib/x__Y.cmo.js bin/z.cmo.js
         ocamlc lib/stubs.o
-      ocamlopt .ppx/js_of_ocaml-ppx/ppx.exe
-        ocamlc lib/.x.objs/x__.{cmi,cmo,cmt}
     ocamlmklib lib/dllx_stubs.so,lib/libx_stubs.a
+      ocamlopt .ppx/js_of_ocaml-ppx/ppx.exe
            ppx lib/x.pp.ml
-           ppx lib/y.pp.ml
-           ppx bin/technologic.pp.ml
-           ppx bin/z.pp.ml
-      ocamlopt lib/.x.objs/x__.{cmx,o}
       ocamldep lib/x.pp.ml.d
+        ocamlc lib/.x.objs/x__.{cmi,cmo,cmt}
+      ocamlopt lib/.x.objs/x__.{cmx,o}
+           ppx lib/y.pp.ml
       ocamldep lib/y.pp.ml.d
+        ocamlc lib/.x.objs/x__Y.{cmi,cmo,cmt}
+      ocamlopt lib/.x.objs/x__Y.{cmx,o}
+           ppx bin/technologic.pp.ml
       ocamldep bin/technologic.pp.ml.d
+           ppx bin/z.pp.ml
       ocamldep bin/z.pp.ml.d
-        ocamlc lib/.x.objs/x__Y.{cmi,cmo,cmt}
    js_of_ocaml .js/js_of_ocaml/js_of_ocaml.cma.js
-   js_of_ocaml .js/stdlib/stdlib.cma.js
-   js_of_ocaml bin/technologic.bc.runtime.js
-      ocamlopt lib/.x.objs/x__Y.{cmx,o}
    js_of_ocaml lib/.x.objs/x__Y.cmo.js
         ocamlc lib/.x.objs/x.{cmi,cmo,cmt}
       ocamlopt lib/.x.objs/x.{cmx,o}
-        ocamlc lib/x.cma
-        ocamlc bin/.technologic.eobjs/z.{cmi,cmo,cmt}
       ocamlopt lib/x.{a,cmxa}
+      ocamlopt lib/x.cmxs
+   js_of_ocaml .js/stdlib/stdlib.cma.js
+   js_of_ocaml bin/technologic.bc.runtime.js
+        ocamlc lib/x.cma
    js_of_ocaml lib/x.cma.js
+        ocamlc bin/.technologic.eobjs/z.{cmi,cmo,cmt}
    js_of_ocaml bin/.technologic.eobjs/z.cmo.js
         ocamlc bin/.technologic.eobjs/technologic.{cmi,cmo,cmt}
-      ocamlopt lib/x.cmxs
    js_of_ocaml bin/.technologic.eobjs/technologic.cmo.js
      jsoo_link bin/technologic.bc.js
   $ $NODE ./_build/default/bin/technologic.bc.js
@@ -37,16 +37,16 @@
   $ $JBUILDER build -j1 --display short --root . bin/technologic.bc.js @install
         ocamlc lib/.x.objs/x__.{cmi,cmo,cmt}
         ocamlc lib/.x.objs/x__Y.{cmi,cmo,cmt}
-      ocamlopt lib/.x.objs/x__.{cmx,o}
         ocamlc lib/.x.objs/x.{cmi,cmo,cmt}
-      ocamlopt lib/.x.objs/x__Y.{cmx,o}
         ocamlc lib/x.cma
-        ocamlc bin/.technologic.eobjs/z.{cmi,cmo,cmt}
+      ocamlopt lib/.x.objs/x__.{cmx,o}
+      ocamlopt lib/.x.objs/x__Y.{cmx,o}
       ocamlopt lib/.x.objs/x.{cmx,o}
-        ocamlc bin/.technologic.eobjs/technologic.{cmi,cmo,cmt}
       ocamlopt lib/x.{a,cmxa}
-        ocamlc bin/technologic.bc
       ocamlopt lib/x.cmxs
+        ocamlc bin/.technologic.eobjs/z.{cmi,cmo,cmt}
+        ocamlc bin/.technologic.eobjs/technologic.{cmi,cmo,cmt}
+        ocamlc bin/technologic.bc
    js_of_ocaml bin/technologic.bc.js
   $ $NODE ./_build/default/bin/technologic.bc.js
   buy it

test/blackbox-tests/test-cases/menhir/run.t

@@ -1,23 +1,23 @@
   $ $JBUILDER build -j1 src/test.exe --display short --root . --debug-dependency-path
       ocamllex src/lexer1.ml
+      ocamldep src/lexer1.ml.d
       ocamllex src/lexer2.ml
+      ocamldep src/lexer2.ml.d
       ocamldep src/test.ml.d
         menhir src/test_base.{ml,mli}
-        menhir src/test_menhir1.{ml,mli}
-      ocamldep src/lexer1.ml.d
-      ocamldep src/lexer2.ml.d
       ocamldep src/test_base.ml.d
+        menhir src/test_menhir1.{ml,mli}
       ocamldep src/test_menhir1.ml.d
       ocamldep src/test_menhir1.mli.d
-      ocamldep src/test_base.mli.d
         ocamlc src/.test.eobjs/test_menhir1.{cmi,cmti}
-        ocamlc src/.test.eobjs/test_base.{cmi,cmti}
         ocamlc src/.test.eobjs/lexer1.{cmi,cmo,cmt}
-      ocamlopt src/.test.eobjs/test_menhir1.{cmx,o}
+      ocamldep src/test_base.mli.d
+        ocamlc src/.test.eobjs/test_base.{cmi,cmti}
         ocamlc src/.test.eobjs/lexer2.{cmi,cmo,cmt}
-      ocamlopt src/.test.eobjs/test_base.{cmx,o}
-      ocamlopt src/.test.eobjs/lexer1.{cmx,o}
         ocamlc src/.test.eobjs/test.{cmi,cmo,cmt}
+      ocamlopt src/.test.eobjs/test_menhir1.{cmx,o}
+      ocamlopt src/.test.eobjs/lexer1.{cmx,o}
+      ocamlopt src/.test.eobjs/test_base.{cmx,o}
       ocamlopt src/.test.eobjs/lexer2.{cmx,o}
       ocamlopt src/.test.eobjs/test.{cmx,o}
       ocamlopt src/test.exe

test/blackbox-tests/test-cases/multiple-private-libs/run.t

@@ -4,13 +4,13 @@ This test checks that there is no clash when two private libraries have the same
           odoc _doc/odoc.css
           odoc _doc/test@a/page-index.odoc
       ocamldep a/test.ml.d
+        ocamlc a/.test.objs/test.{cmi,cmo,cmt}
+          odoc _doc/test@a/test.odoc
+          odoc _doc/test@a/index.html
           odoc _doc/test@b/page-index.odoc
       ocamldep b/test.ml.d
-        ocamlc a/.test.objs/test.{cmi,cmo,cmt}
         ocamlc b/.test.objs/test.{cmi,cmo,cmt}
-          odoc _doc/test@a/test.odoc
           odoc _doc/test@b/test.odoc
-          odoc _doc/test@a/index.html
-          odoc _doc/test@a/Test/.jbuilder-keep,_doc/test@a/Test/index.html
           odoc _doc/test@b/index.html
+          odoc _doc/test@a/Test/.jbuilder-keep,_doc/test@a/Test/index.html
           odoc _doc/test@b/Test/.jbuilder-keep,_doc/test@b/Test/index.html

test/blackbox-tests/test-cases/odoc-unique-mlds/run.t

@@ -4,15 +4,15 @@ Duplicate mld's in the same scope
           odoc _doc/root.lib1/page-index.odoc
           odoc _doc/root.lib1/page-test.odoc
         ocamlc lib1/.root_lib1.objs/root_lib1.{cmi,cmo,cmt}
+          odoc _doc/root.lib1/root_lib1.odoc
+          odoc _doc/root.lib1/index.html
           odoc _doc/root.lib2/page-index.odoc
           odoc _doc/root.lib2/page-test.odoc
         ocamlc lib2/.root_lib2.objs/root_lib2.{cmi,cmo,cmt}
-          odoc _doc/root.lib1/root_lib1.odoc
           odoc _doc/root.lib2/root_lib2.odoc
-          odoc _doc/root.lib1/index.html
+          odoc _doc/root.lib2/index.html
           odoc _doc/root.lib1/test.html
           odoc _doc/root.lib1/Root_lib1/.jbuilder-keep,_doc/root.lib1/Root_lib1/index.html
-          odoc _doc/root.lib2/index.html
           odoc _doc/root.lib2/test.html
           odoc _doc/root.lib2/Root_lib2/.jbuilder-keep,_doc/root.lib2/Root_lib2/index.html
 
@@ -23,14 +23,14 @@ Duplicate mld's in different scope
           odoc _doc/scope1/page-foo.odoc
           odoc _doc/scope1/page-index.odoc
         ocamlc scope1/.scope1.objs/scope1.{cmi,cmo,cmt}
+          odoc _doc/scope1/scope1.odoc
+          odoc _doc/scope1/foo.html
           odoc _doc/scope2/page-foo.odoc
           odoc _doc/scope2/page-index.odoc
         ocamlc scope2/.scope2.objs/scope2.{cmi,cmo,cmt}
-          odoc _doc/scope1/scope1.odoc
           odoc _doc/scope2/scope2.odoc
-          odoc _doc/scope1/foo.html
+          odoc _doc/scope2/foo.html
           odoc _doc/scope1/index.html
           odoc _doc/scope1/Scope1/.jbuilder-keep,_doc/scope1/Scope1/index.html
-          odoc _doc/scope2/foo.html
           odoc _doc/scope2/index.html
           odoc _doc/scope2/Scope2/.jbuilder-keep,_doc/scope2/Scope2/index.html

test/blackbox-tests/test-cases/odoc/run.t

@@ -1,21 +1,21 @@
   $ $JBUILDER build @doc -j1 --display short --root .
       ocamldep foo_byte.ml.d
+        ocamlc .foo_byte.objs/foo_byte.{cmi,cmo,cmt}
+          odoc _doc/foo.byte/foo_byte.odoc
           odoc _doc/foo.byte/page-index.odoc
           odoc _doc/foo.byte/page-test.odoc
+          odoc _doc/foo.byte/Foo_byte/.jbuilder-keep,_doc/foo.byte/Foo_byte/index.html
       ocamldep foo.ml.d
+        ocamlc .foo.objs/foo.{cmi,cmo,cmt}
+          odoc _doc/foo/foo.odoc
           odoc _doc/foo/page-index.odoc
           odoc _doc/foo/page-test.odoc
+          odoc _doc/foo/Foo/.jbuilder-keep,_doc/foo/Foo/index.html
           odoc _doc/odoc.css
-        ocamlc .foo_byte.objs/foo_byte.{cmi,cmo,cmt}
-        ocamlc .foo.objs/foo.{cmi,cmo,cmt}
-          odoc _doc/foo.byte/foo_byte.odoc
-          odoc _doc/foo/foo.odoc
           odoc _doc/foo.byte/index.html
           odoc _doc/foo.byte/test.html
-          odoc _doc/foo.byte/Foo_byte/.jbuilder-keep,_doc/foo.byte/Foo_byte/index.html
           odoc _doc/foo/index.html
           odoc _doc/foo/test.html
-          odoc _doc/foo/Foo/.jbuilder-keep,_doc/foo/Foo/index.html
   $ $JBUILDER runtest -j1 --display short --root .
   <!DOCTYPE html>
   <html xmlns="http://www.w3.org/1999/xhtml">

test/blackbox-tests/test-cases/redirections/run.t

@@ -1,8 +1,8 @@
   $ $JBUILDER runtest -j1 --display short --root . 2>&1 | sed "s/ cmd /  sh /"
             sh stderr,stdout
-            sh both
             sh stderr,stdout
-            sh both
           diff alias runtest
+            sh both
+            sh both
           diff alias runtest
           diff alias runtest