Skip to content

Commit ad64194

Browse files
maxim-uvarovmaxim-uvarov
authored
Merge pull request #35 from nushell-prophet/refactor/integration-test-variants
refactor: standardize integration tests with run-snapshot-test helper
2 parents 0ad9353 + 288b808 commit ad64194

7 files changed

Lines changed: 92 additions & 80 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,8 @@ dotnu dependencies --help
222222

223223
```nushell
224224
dotnu filter-commands-with-no-tests --help
225-
# => Filter commands after `dotnu dependencies` that aren't used by any other command containing `test` in its name.
225+
# => Filter commands after `dotnu dependencies` that aren't used by any test command.
226+
# => Test commands are detected by: name contains 'test' OR file matches 'test*.nu'
226227
# =>
227228
# => Usage:
228229
# => > filter-commands-with-no-tests

‎dotnu/commands.nu‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ export def 'dependencies' [
1010
--definitions-only # output only commands' names definitions
1111
] {
1212
let callees_to_merge = $paths
13+
| sort # ensure consistent order across platforms
1314
| each {
1415
list-module-commands $in --keep-builtins=$keep_builtins --definitions-only=$definitions_only
1516
}
@@ -544,10 +545,9 @@ export def list-module-commands [
544545
}
545546
| where caller != null and caller !~ '^@' # exclude tokens inside attribute blocks
546547
| where shape in ['shape_internalcall' 'shape_external']
548+
| where content not-in (help commands | where command_type == 'keyword' | get name) # always exclude keywords (def, export def, etc.)
547549
| if $keep_builtins { } else {
548-
where content not-in (
549-
help commands | where command_type in ['built-in' 'keyword'] | get name
550-
)
550+
where content not-in (help commands | where command_type == 'built-in' | get name)
551551
}
552552
| select caller content filename_of_caller
553553
| rename --column {content: callee}

‎tests/assets/dotnu-capture-clean.nu‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# open ([tests assets dotnu-capture.nu] | path join)
2+
# | dotnu embeds-remove
13
# this is a typical nushell script
24
# embeds in this script can be updated using command:
35
# `dotnu embeds-update dotnu-capture.nu`

‎tests/assets/dotnu-capture-updated.nu‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# dotnu embeds-update ([tests assets dotnu-capture.nu] | path join) --echo
12
# this is a typical nushell script
23
# embeds in this script can be updated using command:
34
# `dotnu embeds-update dotnu-capture.nu`
@@ -14,7 +15,7 @@ ls | sort-by modified -r | last 2 | print $in
1415
# => ╰───┴──────────────────────────────┴──────┴───────┴────────────╯
1516

1617
random int | print $in
17-
# => 5037388629847034664
18+
# => 4577019228065240114
1819

1920
'Say hello to the core team of the Nushell'
2021
| str replace 'Nushell' 'Best shell'

‎tests/output-yaml/coverage-untested.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# # Public API from mod.nu
2+
# let public_api = open ([dotnu mod.nu] | path join)
3+
# | lines
4+
# | where $it =~ '^\s+"'
5+
# | each { $in | str trim | str replace -r '^"([^"]+)".*' '$1' }
6+
#
7+
# # Find untested commands
8+
# let untested = ["dotnu/*.nu" "tests/test_commands.nu" "toolkit.nu"]
9+
# | each { glob $in }
10+
# | flatten
11+
# | dependencies ...$in
12+
# | filter-commands-with-no-tests
13+
# | where caller in $public_api
14+
# | select caller
15+
#
16+
# # Output as yaml
17+
# {
18+
# public_api_count: ($public_api | length)
19+
# tested_count: (($public_api | length) - ($untested | length))
20+
# untested: ($untested | get caller)
21+
# }
22+
# | to yaml
23+
public_api_count: 13
24+
tested_count: 10
25+
untested:
26+
- embed-add
27+
- embeds-capture-start
28+
- embeds-capture-stop

tests/output-yaml/dependencies --keep_bulitins.yaml renamed to tests/output-yaml/dependencies --keep_builtins.yaml

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,10 @@
11
# glob ([tests assets b *] | path join | str replace -a '\' '/')
22
# | dependencies ...$in --keep-builtins
33
# | to yaml
4-
- caller: example-mod1
5-
callee: export def
6-
filename_of_caller: example-mod1.nu
7-
step: 0
8-
- caller: lscustom
9-
callee: export def
10-
filename_of_caller: example-mod1.nu
11-
step: 0
124
- caller: lscustom
135
callee: ls
146
filename_of_caller: example-mod1.nu
157
step: 0
16-
- caller: command-5
17-
callee: export def
18-
filename_of_caller: example-mod1.nu
19-
step: 0
208
- caller: command-5
219
callee: command-3
2210
filename_of_caller: example-mod1.nu
@@ -29,18 +17,10 @@
2917
callee: append-random
3018
filename_of_caller: example-mod1.nu
3119
step: 0
32-
- caller: sort-by-custom
33-
callee: def
34-
filename_of_caller: example-mod1.nu
35-
step: 0
3620
- caller: sort-by-custom
3721
callee: sort-by
3822
filename_of_caller: example-mod1.nu
3923
step: 0
40-
- caller: command-3
41-
callee: def
42-
filename_of_caller: example-mod1.nu
43-
step: 0
4424
- caller: command-3
4525
callee: lscustom
4626
filename_of_caller: example-mod1.nu
@@ -49,10 +29,6 @@
4929
callee: sort-by-custom
5030
filename_of_caller: example-mod1.nu
5131
step: 0
52-
- caller: first-custom
53-
callee: def
54-
filename_of_caller: example-mod1.nu
55-
step: 0
5632
- caller: first-custom
5733
callee: first
5834
filename_of_caller: example-mod1.nu
@@ -61,9 +37,9 @@
6137
callee: select
6238
filename_of_caller: example-mod1.nu
6339
step: 0
64-
- caller: append-random
65-
callee: export def
66-
filename_of_caller: example-mod2.nu
40+
- caller: example-mod1
41+
filename_of_caller: example-mod1.nu
42+
callee: null
6743
step: 0
6844
- caller: append-random
6945
callee: append
@@ -73,10 +49,6 @@
7349
callee: random chars
7450
filename_of_caller: example-mod2.nu
7551
step: 0
76-
- caller: command-5
77-
callee: def
78-
step: 1
79-
filename_of_caller: example-mod1.nu
8052
- caller: command-5
8153
callee: lscustom
8254
step: 1
@@ -101,10 +73,6 @@
10173
callee: random chars
10274
step: 1
10375
filename_of_caller: example-mod1.nu
104-
- caller: command-3
105-
callee: export def
106-
step: 1
107-
filename_of_caller: example-mod1.nu
10876
- caller: command-3
10977
callee: ls
11078
step: 1

‎toolkit.nu‎

Lines changed: 52 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ export def 'main test-unit' [
2828
}
2929

3030
# Run integration tests
31+
#
32+
# These are snapshot tests: each test runs a command and saves the output to a file.
33+
# The files are committed to git, so `git diff` reveals any behavioral changes.
34+
# The `run-snapshot-test` helper embeds the generating code as header comments,
35+
# making each snapshot self-documenting.
3136
export def 'main test-integration' [
3237
--json # output results as JSON for external consumption
3338
] {
@@ -36,6 +41,7 @@ export def 'main test-integration' [
3641
(test-dependencies-keep_builtins)
3742
(test-embeds-remove)
3843
(test-embeds-update)
44+
(test-coverage)
3945
]
4046
# Run numd on README if available
4147
| if (scope modules | where name == 'numd' | is-not-empty) {
@@ -44,19 +50,11 @@ export def 'main test-integration' [
4450
| if $json { to json --raw } else { }
4551
}
4652

47-
# Test dependencies command
48-
def 'test-dependencies' [] {
49-
let output_file = ['tests' 'output-yaml' 'dependencies.yaml'] | path join
50-
53+
# Run command and save output with source code as header comment
54+
def run-snapshot-test [name: string output_file: string command_src: closure] {
5155
mkdir ($output_file | path dirname)
5256
rm -f $output_file
5357

54-
let command_src = {
55-
glob ([tests assets b *] | path join | str replace -a '\' '/')
56-
| dependencies ...$in
57-
| to yaml
58-
}
59-
6058
let command_text = view source $command_src
6159
| lines | skip | drop | str trim
6260
| each { $'# ($in)' }
@@ -65,54 +63,68 @@ def 'test-dependencies' [] {
6563
$command_text + (char nl) + (do $command_src)
6664
| save -f $output_file
6765

68-
{test: 'dependencies' file: $output_file}
66+
{test: $name file: $output_file}
67+
}
68+
69+
# Test dependencies command
70+
def 'test-dependencies' [] {
71+
run-snapshot-test 'dependencies' ([tests output-yaml dependencies.yaml] | path join) {
72+
glob ([tests assets b *] | path join | str replace -a '\' '/')
73+
| dependencies ...$in
74+
| to yaml
75+
}
6976
}
7077

7178
# Test dependencies command with keep-builtins option
7279
def 'test-dependencies-keep_builtins' [] {
73-
let output_file = ['tests' 'output-yaml' 'dependencies --keep_bulitins.yaml'] | path join
74-
75-
mkdir ($output_file | path dirname)
76-
rm -f $output_file
77-
78-
let command_src = {
80+
run-snapshot-test 'dependencies --keep-builtins' ([tests output-yaml 'dependencies --keep_builtins.yaml'] | path join) {
7981
glob ([tests assets b *] | path join | str replace -a '\' '/')
8082
| dependencies ...$in --keep-builtins
8183
| to yaml
8284
}
83-
84-
let command_text = view source $command_src
85-
| lines | skip | drop | str trim
86-
| each { $'# ($in)' }
87-
| str join (char nl)
88-
89-
$command_text + (char nl) + (do $command_src)
90-
| save -f $output_file
91-
92-
{test: 'dependencies --keep-builtins' file: $output_file}
9385
}
9486

9587
# Test embeds-remove command
9688
def 'test-embeds-remove' [] {
97-
let input_file = 'tests/assets/dotnu-capture.nu'
98-
let output_file = 'tests/assets/dotnu-capture-clean.nu'
99-
100-
open $input_file
101-
| dotnu embeds-remove
102-
| save -f $output_file
103-
104-
{test: 'embeds-remove' file: $output_file}
89+
run-snapshot-test 'embeds-remove' ([tests assets dotnu-capture-clean.nu] | path join) {
90+
open ([tests assets dotnu-capture.nu] | path join)
91+
| dotnu embeds-remove
92+
}
10593
}
10694

10795
# Test embeds-update command
10896
def 'test-embeds-update' [] {
109-
let input_file = 'tests/assets/dotnu-capture.nu'
110-
let output_file = 'tests/assets/dotnu-capture-updated.nu'
97+
run-snapshot-test 'embeds-update' ([tests assets dotnu-capture-updated.nu] | path join) {
98+
dotnu embeds-update ([tests assets dotnu-capture.nu] | path join) --echo
99+
}
100+
}
111101

112-
dotnu embeds-update $input_file --echo
113-
| save -f $output_file
102+
# Test coverage: find public API commands without tests
103+
def 'test-coverage' [] {
104+
run-snapshot-test 'coverage' ([tests output-yaml coverage-untested.yaml] | path join) {
105+
# Public API from mod.nu
106+
let public_api = open ([dotnu mod.nu] | path join)
107+
| lines
108+
| where $it =~ '^\s+"'
109+
| each { $in | str trim | str replace -r '^"([^"]+)".*' '$1' }
114110

115-
{test: 'embeds-update' file: $output_file}
111+
# Find untested commands
112+
let untested = ["dotnu/*.nu" "tests/test_commands.nu" "toolkit.nu"]
113+
| each { glob $in }
114+
| flatten
115+
| dependencies ...$in
116+
| filter-commands-with-no-tests
117+
| where caller in $public_api
118+
| select caller
119+
120+
# Output as yaml
121+
{
122+
public_api_count: ($public_api | length)
123+
tested_count: (($public_api | length) - ($untested | length))
124+
untested: ($untested | get caller)
125+
}
126+
| to yaml
127+
}
116128
}
117129

118130
# Test numd on README

0 commit comments

Comments
 (0)