Skip to content

Commit

Permalink
Merge pull request #5144 from sellout/group-test-results
Browse files Browse the repository at this point in the history
  • Loading branch information
@aryairani
aryairani authored Jun 27, 2024
2 parents 4d8fa73 + 1a753ff commit f824743
Show file tree
Hide file tree
Showing 27 changed files with 380 additions and 372 deletions.
68 changes: 38 additions & 30 deletions unison-cli/src/Unison/Codebase/Editor/HandleInput/Tests.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import Data.Map qualified as Map
import Data.Set qualified as Set
import Data.Set.NonEmpty (NESet)
import Data.Set.NonEmpty qualified as NESet
import Data.Tuple qualified as Tuple
import Unison.ABT qualified as ABT
import Unison.Builtin.Decls qualified as DD
import Unison.Cli.Monad (Cli)
Expand Down Expand Up @@ -69,21 +68,24 @@ handleTest TestInput {includeLibNamespace, path, showFailures, showSuccesses} =
Map.fromList <$> Cli.runTransaction do
Set.toList testRefs & wither \case
rid -> fmap (rid,) <$> Codebase.getWatch codebase WK.TestWatch rid
let (oks, fails) = passFails cachedTests
passFails :: (Ord r) => Map r (Term v a) -> ([(r, Text)], [(r, Text)])
passFails = Tuple.swap . partitionEithers . concat . map p . Map.toList
let (fails, oks) = passFails cachedTests
passFails :: (Ord r) => Map r (Term v a) -> (Map r [Text], Map r [Text])
passFails =
Map.foldrWithKey
(\r v (f, o) -> bimap (\ts -> if null ts then f else Map.insert r ts f) (\ts -> if null ts then o else Map.insert r ts o) . partitionEithers $ p v)
(Map.empty, Map.empty)
where
p :: (r, Term v a) -> [Either (r, Text) (r, Text)]
p (r, tm) = case tm of
Term.List' ts -> mapMaybe (q r) (toList ts)
p :: Term v a -> [Either Text Text]
p = \case
Term.List' ts -> mapMaybe q $ toList ts
_ -> []
q r = \case
q = \case
Term.App' (Term.Constructor' (ConstructorReference ref cid)) (Term.Text' msg) ->
if
| ref == DD.testResultRef ->
if
| cid == DD.okConstructorId -> Just (Right (r, msg))
| cid == DD.failConstructorId -> Just (Left (r, msg))
| cid == DD.okConstructorId -> Just (Right msg)
| cid == DD.failConstructorId -> Just (Left msg)
| otherwise -> Nothing
| otherwise -> Nothing
_ -> Nothing
Expand Down Expand Up @@ -123,7 +125,7 @@ handleTest TestInput {includeLibNamespace, path, showFailures, showSuccesses} =
pure [(r, tm')]

let m = Map.fromList computedTests
(mOks, mFails) = passFails m
(mFails, mOks) = passFails m
Cli.respondNumbered $ TestResults Output.NewlyComputed fqnPPE showSuccesses showFailures mOks mFails

handleIOTest :: HQ.HashQualified Name -> Cli ()
Expand All @@ -135,10 +137,14 @@ handleIOTest main = do
let isIOTest typ = Foldable.any (Typechecker.isSubtype typ) $ Runtime.ioTestTypes runtime
refs <- resolveHQNames names (Set.singleton main)
(fails, oks) <-
refs & foldMapM \(ref, typ) -> do
when (not $ isIOTest typ) do
Cli.returnEarly (BadMainFunction "io.test" main typ suffixifiedPPE (Foldable.toList $ Runtime.ioTestTypes runtime))
runIOTest suffixifiedPPE ref
Foldable.foldrM
( \(ref, typ) (f, o) -> do
when (not $ isIOTest typ) $
Cli.returnEarly (BadMainFunction "io.test" main typ suffixifiedPPE (Foldable.toList $ Runtime.ioTestTypes runtime))
bimap (\ts -> if null ts then f else Map.insert ref ts f) (\ts -> if null ts then o else Map.insert ref ts o) <$> runIOTest suffixifiedPPE ref
)
(Map.empty, Map.empty)
refs
Cli.respondNumbered $ TestResults Output.NewlyComputed suffixifiedPPE True True oks fails

findTermsOfTypes :: Codebase.Codebase m Symbol Ann -> Bool -> Path -> NESet (Type.Type Symbol Ann) -> Cli (Set TermReferenceId)
Expand All @@ -163,15 +169,20 @@ handleAllIOTests = do
let suffixifiedPPE = PPED.suffixifiedPPE pped
ioTestRefs <- findTermsOfTypes codebase False Path.empty (Runtime.ioTestTypes runtime)
case NESet.nonEmptySet ioTestRefs of
Nothing -> Cli.respondNumbered $ TestResults Output.NewlyComputed suffixifiedPPE True True [] []
Nothing -> Cli.respondNumbered $ TestResults Output.NewlyComputed suffixifiedPPE True True Map.empty Map.empty
Just neTestRefs -> do
let total = NESet.size neTestRefs
(fails, oks) <-
toList neTestRefs & zip [1 :: Int ..] & foldMapM \(n, r) -> do
Cli.respond $ TestIncrementalOutputStart suffixifiedPPE (n, total) r
(fails, oks) <- runIOTest suffixifiedPPE r
Cli.respond $ TestIncrementalOutputEnd suffixifiedPPE (n, total) r (null fails)
pure (fails, oks)
toList neTestRefs
& zip [1 :: Int ..]
& Foldable.foldrM
( \(n, r) (f, o) -> do
Cli.respond $ TestIncrementalOutputStart suffixifiedPPE (n, total) r
(fails, oks) <- runIOTest suffixifiedPPE r
Cli.respond $ TestIncrementalOutputEnd suffixifiedPPE (n, total) r (null fails)
pure (if null fails then f else Map.insert r fails f, if null oks then o else Map.insert r oks o)
)
(Map.empty, Map.empty)
Cli.respondNumbered $ TestResults Output.NewlyComputed suffixifiedPPE True True oks fails

resolveHQNames :: Names -> Set (HQ.HashQualified Name) -> Cli (Set (Reference.Id, Type.Type Symbol Ann))
Expand All @@ -197,28 +208,25 @@ resolveHQNames parseNames hqNames =
typ <- MaybeT (Codebase.getTypeOfReferent codebase (Referent.fromTermReferenceId ref))
pure (ref, typ)

runIOTest :: PPE.PrettyPrintEnv -> Reference.Id -> Cli ([(Reference.Id, Text)], [(Reference.Id, Text)])
runIOTest :: PPE.PrettyPrintEnv -> Reference.Id -> Cli ([Text], [Text])
runIOTest ppe ref = do
let a = ABT.annotation tm
tm = DD.forceTerm a a (Term.refId a ref)
-- Don't cache IO tests
tm' <- RuntimeUtils.evalUnisonTerm False ppe False tm
pure $ partitionTestResults [(ref, tm')]
pure $ partitionTestResults tm'

partitionTestResults ::
[(Reference.Id, Term Symbol Ann)] ->
([(Reference.Id, Text {- fails -})], [(Reference.Id, Text {- oks -})])
partitionTestResults results = fold $ do
(ref, tm) <- results
partitionTestResults :: Term Symbol Ann -> ([Text {- fails -}], [Text {- oks -}])
partitionTestResults tm = fold $ do
element <- case tm of
Term.List' ts -> toList ts
_ -> empty
case element of
Term.App' (Term.Constructor' (ConstructorReference conRef cid)) (Term.Text' msg) -> do
guard (conRef == DD.testResultRef)
if
| cid == DD.okConstructorId -> pure (mempty, [(ref, msg)])
| cid == DD.failConstructorId -> pure ([(ref, msg)], mempty)
| cid == DD.okConstructorId -> pure (mempty, [msg])
| cid == DD.failConstructorId -> pure ([msg], mempty)
| otherwise -> empty
_ -> empty

Expand Down
4 changes: 2 additions & 2 deletions unison-cli/src/Unison/Codebase/Editor/Output.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ data NumberedOutput
PPE.PrettyPrintEnv
ShowSuccesses
ShowFailures
[(TermReferenceId, Text)] -- oks
[(TermReferenceId, Text)] -- fails
(Map TermReferenceId [Text]) -- oks
(Map TermReferenceId [Text]) -- fails
| Output'Todo !TodoOutput
| -- | CantDeleteDefinitions ppe couldntDelete becauseTheseStillReferenceThem
CantDeleteDefinitions PPE.PrettyPrintEnvDecl (Map LabeledDependency (NESet LabeledDependency))
Expand Down
16 changes: 8 additions & 8 deletions unison-cli/src/Unison/CommandLine/OutputMessages.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,8 @@ notifyNumbered = \case
)
(showDiffNamespace ShowNumbers ppe (absPathToBranchId bAbs) (absPathToBranchId bAbs) diff)
TestResults stats ppe _showSuccess _showFailures oksUnsorted failsUnsorted ->
let oks = Name.sortByText (HQ.toText . fst) [(name r, msg) | (r, msg) <- oksUnsorted]
fails = Name.sortByText (HQ.toText . fst) [(name r, msg) | (r, msg) <- failsUnsorted]
let oks = Name.sortByText (HQ.toText . fst) [(name r, msgs) | (r, msgs) <- Map.toList oksUnsorted]
fails = Name.sortByText (HQ.toText . fst) [(name r, msgs) | (r, msgs) <- Map.toList failsUnsorted]
name r = PPE.termName ppe (Referent.fromTermReferenceId r)
in ( case stats of
CachedTests 0 _ -> P.callout "😶" $ "No tests to run."
Expand Down Expand Up @@ -2535,8 +2535,8 @@ displayRendered outputLoc pp =

displayTestResults ::
Bool -> -- whether to show the tip
[(HQ.HashQualified Name, Text)] ->
[(HQ.HashQualified Name, Text)] ->
[(HQ.HashQualified Name, [Text])] ->
[(HQ.HashQualified Name, [Text])] ->
Pretty
displayTestResults showTip oks fails =
let name = P.text . HQ.toText
Expand All @@ -2545,23 +2545,23 @@ displayTestResults showTip oks fails =
then mempty
else
P.indentN 2 $
P.numberedColumn2ListFrom 0 [(P.green "◉ " <> name r, " " <> P.green (P.text msg)) | (r, msg) <- oks]
P.numberedColumn2ListFrom 0 [(name r, P.lines $ P.green . (" ◉ " <>) . P.text <$> msgs) | (r, msgs) <- oks]
okSummary =
if null oks
then mempty
else "✅ " <> P.bold (P.num (length oks)) <> P.green " test(s) passing"
else "✅ " <> P.bold (P.num (sum $ fmap (length . snd) oks)) <> P.green " test(s) passing"
failMsg =
if null fails
then mempty
else
P.indentN 2 $
P.numberedColumn2ListFrom
(length oks)
[(P.red "✗ " <> name r, " " <> P.red (P.text msg)) | (r, msg) <- fails]
[(name r, P.lines $ P.red . (" ✗ " <>) . P.text <$> msgs) | (r, msgs) <- fails]
failSummary =
if null fails
then mempty
else "🚫 " <> P.bold (P.num (length fails)) <> P.red " test(s) failing"
else "🚫 " <> P.bold (P.num (sum $ fmap (length . snd) fails)) <> P.red " test(s) failing"
tipMsg =
if not showTip || (null oks && null fails)
then mempty
Expand Down
2 changes: 1 addition & 1 deletion unison-src/transcripts-manual/docs.to-html.output.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
I'll now fetch the latest version of the base Unison
library...
Downloaded 14053 entities.
Downloaded 14067 entities.
🎨 Type `ui` to explore this project's code in your browser.
🔭 Discover libraries at https://share.unison-lang.org
Expand Down
4 changes: 2 additions & 2 deletions unison-src/transcripts-using-base/_base.output.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ testAutoClean _ =
New test results:
1. ◉ testAutoClean our temporary directory should exist
2. ◉ testAutoClean our temporary directory should no longer exist
1. testAutoClean ◉ our temporary directory should exist
◉ our temporary directory should no longer exist
✅ 2 test(s) passing
Expand Down
136 changes: 68 additions & 68 deletions unison-src/transcripts-using-base/binary-encoding-nats.output.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,74 +95,74 @@ testABunchOfNats _ =
New test results:
1. ◉ testABunchOfNats successfully decoded 4294967295 using 64 bit Big Endian
2. ◉ testABunchOfNats consumed all input
3. ◉ testABunchOfNats successfully decoded 4294967295 using 64 bit Little Endian
4. ◉ testABunchOfNats consumed all input
5. ◉ testABunchOfNats successfully decoded 4294967295 using 32 bit Big Endian
6. ◉ testABunchOfNats consumed all input
7. ◉ testABunchOfNats successfully decoded 4294967295 using 32 bit Little Endian
8. ◉ testABunchOfNats consumed all input
9. ◉ testABunchOfNats successfully decoded 1090519040 using 64 bit Big Endian
10. ◉ testABunchOfNats consumed all input
11. ◉ testABunchOfNats successfully decoded 1090519040 using 64 bit Little Endian
12. ◉ testABunchOfNats consumed all input
13. ◉ testABunchOfNats successfully decoded 1090519040 using 32 bit Big Endian
14. ◉ testABunchOfNats consumed all input
15. ◉ testABunchOfNats successfully decoded 1090519040 using 32 bit Little Endian
16. ◉ testABunchOfNats consumed all input
17. ◉ testABunchOfNats successfully decoded 4259840 using 64 bit Big Endian
18. ◉ testABunchOfNats consumed all input
19. ◉ testABunchOfNats successfully decoded 4259840 using 64 bit Little Endian
20. ◉ testABunchOfNats consumed all input
21. ◉ testABunchOfNats successfully decoded 4259840 using 32 bit Big Endian
22. ◉ testABunchOfNats consumed all input
23. ◉ testABunchOfNats successfully decoded 4259840 using 32 bit Little Endian
24. ◉ testABunchOfNats consumed all input
25. ◉ testABunchOfNats successfully decoded 16640 using 64 bit Big Endian
26. ◉ testABunchOfNats consumed all input
27. ◉ testABunchOfNats successfully decoded 16640 using 64 bit Little Endian
28. ◉ testABunchOfNats consumed all input
29. ◉ testABunchOfNats successfully decoded 16640 using 32 bit Big Endian
30. ◉ testABunchOfNats consumed all input
31. ◉ testABunchOfNats successfully decoded 16640 using 32 bit Little Endian
32. ◉ testABunchOfNats consumed all input
33. ◉ testABunchOfNats successfully decoded 16640 using 16 bit Big Endian
34. ◉ testABunchOfNats consumed all input
35. ◉ testABunchOfNats successfully decoded 16640 using 16 bit Little Endian
36. ◉ testABunchOfNats consumed all input
37. ◉ testABunchOfNats successfully decoded 2255827097 using 64 bit Big Endian
38. ◉ testABunchOfNats consumed all input
39. ◉ testABunchOfNats successfully decoded 2255827097 using 64 bit Little Endian
40. ◉ testABunchOfNats consumed all input
41. ◉ testABunchOfNats successfully decoded 2255827097 using 32 bit Big Endian
42. ◉ testABunchOfNats consumed all input
43. ◉ testABunchOfNats successfully decoded 2255827097 using 32 bit Little Endian
44. ◉ testABunchOfNats consumed all input
45. ◉ testABunchOfNats successfully decoded 65 using 64 bit Big Endian
46. ◉ testABunchOfNats consumed all input
47. ◉ testABunchOfNats successfully decoded 65 using 64 bit Little Endian
48. ◉ testABunchOfNats consumed all input
49. ◉ testABunchOfNats successfully decoded 65 using 32 bit Big Endian
50. ◉ testABunchOfNats consumed all input
51. ◉ testABunchOfNats successfully decoded 65 using 32 bit Little Endian
52. ◉ testABunchOfNats consumed all input
53. ◉ testABunchOfNats successfully decoded 65 using 16 bit Big Endian
54. ◉ testABunchOfNats consumed all input
55. ◉ testABunchOfNats successfully decoded 65 using 16 bit Little Endian
56. ◉ testABunchOfNats consumed all input
57. ◉ testABunchOfNats successfully decoded 0 using 64 bit Big Endian
58. ◉ testABunchOfNats consumed all input
59. ◉ testABunchOfNats successfully decoded 0 using 64 bit Little Endian
60. ◉ testABunchOfNats consumed all input
61. ◉ testABunchOfNats successfully decoded 0 using 32 bit Big Endian
62. ◉ testABunchOfNats consumed all input
63. ◉ testABunchOfNats successfully decoded 0 using 32 bit Little Endian
64. ◉ testABunchOfNats consumed all input
65. ◉ testABunchOfNats successfully decoded 0 using 16 bit Big Endian
66. ◉ testABunchOfNats consumed all input
67. ◉ testABunchOfNats successfully decoded 0 using 16 bit Little Endian
68. ◉ testABunchOfNats consumed all input
1. testABunchOfNats ◉ successfully decoded 4294967295 using 64 bit Big Endian
◉ consumed all input
◉ successfully decoded 4294967295 using 64 bit Little Endian
◉ consumed all input
◉ successfully decoded 4294967295 using 32 bit Big Endian
◉ consumed all input
◉ successfully decoded 4294967295 using 32 bit Little Endian
◉ consumed all input
◉ successfully decoded 1090519040 using 64 bit Big Endian
◉ consumed all input
◉ successfully decoded 1090519040 using 64 bit Little Endian
◉ consumed all input
◉ successfully decoded 1090519040 using 32 bit Big Endian
◉ consumed all input
◉ successfully decoded 1090519040 using 32 bit Little Endian
◉ consumed all input
◉ successfully decoded 4259840 using 64 bit Big Endian
◉ consumed all input
◉ successfully decoded 4259840 using 64 bit Little Endian
◉ consumed all input
◉ successfully decoded 4259840 using 32 bit Big Endian
◉ consumed all input
◉ successfully decoded 4259840 using 32 bit Little Endian
◉ consumed all input
◉ successfully decoded 16640 using 64 bit Big Endian
◉ consumed all input
◉ successfully decoded 16640 using 64 bit Little Endian
◉ consumed all input
◉ successfully decoded 16640 using 32 bit Big Endian
◉ consumed all input
◉ successfully decoded 16640 using 32 bit Little Endian
◉ consumed all input
◉ successfully decoded 16640 using 16 bit Big Endian
◉ consumed all input
◉ successfully decoded 16640 using 16 bit Little Endian
◉ consumed all input
◉ successfully decoded 2255827097 using 64 bit Big Endian
◉ consumed all input
◉ successfully decoded 2255827097 using 64 bit Little Endian
◉ consumed all input
◉ successfully decoded 2255827097 using 32 bit Big Endian
◉ consumed all input
◉ successfully decoded 2255827097 using 32 bit Little Endian
◉ consumed all input
◉ successfully decoded 65 using 64 bit Big Endian
◉ consumed all input
◉ successfully decoded 65 using 64 bit Little Endian
◉ consumed all input
◉ successfully decoded 65 using 32 bit Big Endian
◉ consumed all input
◉ successfully decoded 65 using 32 bit Little Endian
◉ consumed all input
◉ successfully decoded 65 using 16 bit Big Endian
◉ consumed all input
◉ successfully decoded 65 using 16 bit Little Endian
◉ consumed all input
◉ successfully decoded 0 using 64 bit Big Endian
◉ consumed all input
◉ successfully decoded 0 using 64 bit Little Endian
◉ consumed all input
◉ successfully decoded 0 using 32 bit Big Endian
◉ consumed all input
◉ successfully decoded 0 using 32 bit Little Endian
◉ consumed all input
◉ successfully decoded 0 using 16 bit Big Endian
◉ consumed all input
◉ successfully decoded 0 using 16 bit Little Endian
◉ consumed all input
✅ 68 test(s) passing
Expand Down
Loading

0 comments on commit f824743

Please sign in to comment.