Skip to content

test(cl): run ';' expect cases and skip output diff only #1743

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
luoliwoshang wants to merge 1 commit into
base: main
Choose a base branch
from
+119 −36
Open

test(cl): run ';' expect cases and skip output diff only #1743

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 32 additions & 6 deletions cl/cltest/cltest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,9 @@ func FromDir(t *testing.T, sel, relDir string) {
}

type runOptions struct {
conf *build.Config
filter func(string) string
conf *build.Config
filter func(string) string
expectedRunError map[string]struct{}
}

// RunOption customizes RunFromDir behavior.
Expand All @@ -100,6 +101,24 @@ func WithOutputFilter(filter func(string) string) RunOption {
}
}

// WithExpectedRunErrors marks relPkg paths that are expected to fail at run/build time.
// Marked cases are still executed:
// - if run/build fails: treated as pass
// - if run/build succeeds: treated as failure ("unexpected pass")
func WithExpectedRunErrors(relPkgs []string) RunOption {
return func(opts *runOptions) {
if len(relPkgs) == 0 {
return
}
if opts.expectedRunError == nil {
opts.expectedRunError = make(map[string]struct{}, len(relPkgs))
}
for _, relPkg := range relPkgs {
opts.expectedRunError[relPkg] = struct{}{}
}
}
}

// FilterEmulatorOutput strips emulator boot logs by returning output after "entry 0x...".
func FilterEmulatorOutput(output string) string {
output = strings.ReplaceAll(output, "\r\n", "\n")
Expand Down Expand Up @@ -150,7 +169,7 @@ func RunFromDir(t *testing.T, sel, relDir string, ignore []string, opts ...RunOp
relPkg = "./" + filepath.ToSlash(relPkg)
if _, ok := ignoreSet[relPkg]; ok {
t.Run(name, func(t *testing.T) {
t.Skip("skip platform-specific output mismatch")
t.Skip("ignored by test configuration")
})
continue
}
Expand Down Expand Up @@ -223,9 +242,7 @@ func testRunFrom(t *testing.T, pkgDir, relPkg, sel string, opts runOptions) {
}
t.Fatal("ReadFile failed:", err)
}
if bytes.Equal(expected, []byte{';'}) { // expected == ";" means skipping expect.txt
return
}
skipDiff := bytes.Equal(expected, []byte{';'}) // expected == ";" means run-only (skip output diff)

var output []byte
if opts.conf != nil {
Expand All @@ -234,9 +251,18 @@ func testRunFrom(t *testing.T, pkgDir, relPkg, sel string, opts runOptions) {
output, err = RunAndCapture(relPkg, pkgDir)
}
if err != nil {
if _, ok := opts.expectedRunError[relPkg]; ok {
return
}
t.Logf("raw output:\n%s", string(output))
t.Fatalf("run failed: %v\noutput: %s", err, string(output))
}
if _, ok := opts.expectedRunError[relPkg]; ok {
t.Fatalf("unexpected pass: %s is marked as expected run error", relPkg)
}
if skipDiff {
return
}
if opts.filter != nil {
output = []byte(opts.filter(string(output)))
}
Expand Down
117 changes: 87 additions & 30 deletions cl/compile_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ var embedTargetConfigs = []embedTargetConfig{
"./_testgo/cursor", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/defer4", // unexpected output: got "fatal error", expected "recover: panic message"
"./_testgo/goexit", // llgo panic: unsatisfied import internal/runtime/sys
"./_testgo/goroutine", // timeout: emulator keeps printing without auto-exit
"./_testgo/indexerr", // unexpected output: len(dst)=12, len(src)=0 (got "fatal error")
"./_testgo/makeslice", // unexpected output: len(dst)=23, len(src)=0 (got "fatal error\\nmust error")
"./_testgo/reflect", // llgo panic: unsatisfied import internal/runtime/sys
Expand Down Expand Up @@ -118,52 +119,95 @@ var embedTargetConfigs = []embedTargetConfig{
target: "esp32",
ignoreByDir: map[string][]string{
"./_testgo": {
"./_testgo/abimethod", // panic: internal/bytealg selected .s files require plan9asm translation
"./_testgo/alias", // unexpected output
"./_testgo/cgodefer", // panic: cannot build SSA for packages
"./_testgo/cgopython", // panic: cannot build SSA for packages
"./_testgo/cursor", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/defer4", // runtime output: fatal error
"./_testgo/indexerr", // runtime output: fatal error
"./_testgo/invoke", // unexpected output
"./_testgo/makeslice", // runtime output: fatal error
"./_testgo/multiret", // unexpected output
"./_testgo/select", // timeout: emulator did not auto-exit
"./_testgo/sigsegv", // unexpected output
"./_testgo/struczero", // timeout: emulator did not auto-exit
"./_testgo/abimethod", // panic: internal/bytealg selected .s files require plan9asm translation
"./_testgo/alias", // unexpected output
"./_testgo/cgobasic", // fast fail: build constraints exclude all Go files (cgo)
"./_testgo/cgocfiles", // fast fail: build constraints exclude all Go files (cgo)
"./_testgo/cgodefer", // panic: cannot build SSA for packages
"./_testgo/cgofull", // fast fail: build constraints exclude all Go files (cgo)
"./_testgo/cgomacro", // fast fail: build constraints exclude all Go files (cgo)
"./_testgo/cgopython", // panic: cannot build SSA for packages
"./_testgo/chan", // timeout: emulator did not auto-exit
"./_testgo/cursor", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/defer3", // runtime output: fatal error (exit status 2)
"./_testgo/defer4", // runtime output: fatal error
"./_testgo/defer5", // runtime output: fatal error (exit status 2)
"./_testgo/goexit", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/goroutine", // timeout: emulator keeps printing without auto-exit
"./_testgo/indexerr", // runtime output: fatal error
"./_testgo/invoke", // unexpected output
"./_testgo/makeslice", // runtime output: fatal error
"./_testgo/multiret", // unexpected output
"./_testgo/reflect", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/reflectconv", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/reflectfn", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/reflectmkfn", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/rewrite", // panic: internal/bytealg: selected .s files require plan9asm translation
"./_testgo/select", // timeout: emulator did not auto-exit
"./_testgo/selects", // timeout: emulator did not auto-exit
"./_testgo/sigsegv", // unexpected output
"./_testgo/struczero", // timeout: emulator did not auto-exit
"./_testgo/syncmap", // panic: internal/bytealg: selected .s files require plan9asm translation
},
"./_testlibc": {
"./_testlibc/atomic", // unexpected output
"./_testlibc/complex", // link error: ld.lld undefined symbol cabsf
"./_testlibc/demangle", // link error: ld.lld unknown argument -Wl,-search_paths_first
"./_testlibc/once", // panic: cannot build SSA for packages
"./_testlibc/setjmp", // link error: ld.lld undefined symbol stderr
"./_testlibc/sqlite", // link error: ld.lld unable to find library -lsqlite3
},
"./_testrt": {
"./_testrt/asmfull", // unexpected output
"./_testrt/cast", // timeout: emulator did not auto-exit
"./_testrt/complex", // unexpected output
"./_testrt/fprintf", // link error: ld.lld undefined symbol __stderrp
"./_testrt/hello", // panic: cannot build SSA for packages
"./_testrt/linkname", // unexpected output
"./_testrt/strlen", // panic: runtime index out of range
"./_testrt/struct", // panic: runtime index out of range
"./_testrt/tpfunc", // unexpected output
"./_testrt/typalias", // panic: runtime index out of range
"./_testrt/asmfull", // unexpected output
"./_testrt/cast", // timeout: emulator did not auto-exit
"./_testrt/complex", // unexpected output
"./_testrt/fprintf", // link error: ld.lld undefined symbol __stderrp
"./_testrt/hello", // panic: cannot build SSA for packages
"./_testrt/linkname", // unexpected output
"./_testrt/strlen", // panic: runtime index out of range
"./_testrt/struct", // panic: runtime index out of range
"./_testrt/tpfunc", // unexpected output
"./_testrt/typalias", // panic: runtime index out of range
"./_testrt/panic", // runtime output: fatal error (exit status 2)
"./_testrt/unreachable", // timeout: emulator panic (Instruction access fault), no auto-exit
"./_testrt/vamethod", // timeout: emulator hangs and does not auto-exit
},
"./_testdata": {
"./_testdata/cpkgimp", // unexpected output
"./_testdata/debug", // panic: internal/bytealg: selected .s files require plan9asm translation
},
},
},
}

var hostIgnoreByDir = map[string][]string{
"./_testgo": {
"./_testgo/goexit", // unexpected run/build failure, temporarily ignored; see https://github.com/goplus/llgo/issues/1745
},
"./_testlibgo": {
"./_testlibgo/waitgroup", // unexpected run/build failure, temporarily ignored; see https://github.com/goplus/llgo/issues/1745
"./_testlibgo/sync", // unexpected run/build failure, temporarily ignored; see https://github.com/goplus/llgo/issues/1745
},
}

var hostExpectedRunErrorByDir = map[string][]string{
"./_testgo": {
"./_testgo/defer3", // panic/exit is expected for this case
"./_testgo/defer5", // panic/exit is expected for this case
},
"./_testlibgo": {
"./_testlibgo/errors", // panic/exit is expected for this case
},
}

func runEmbedTargetSuite(t *testing.T, target, relDir string, ignore []string) {
t.Helper()
conf := build.NewDefaultConf(build.ModeRun)
conf.Target = target
conf.Emulator = true
conf.ForceRebuild = true
// Embedded emulator exit codes are currently not reliable enough to
// distinguish panic/crash from normal completion, so expected-run-error
// semantics are intentionally disabled for embedded targets for now.
cltest.RunFromDir(t, "", relDir, ignore,
cltest.WithRunConfig(conf),
cltest.WithOutputFilter(cltest.FilterEmulatorOutput),
Expand All @@ -175,7 +219,10 @@ func TestFromTestgo(t *testing.T) {
}

func TestRunFromTestgo(t *testing.T) {
cltest.RunFromDir(t, "", "./_testgo", nil)
relDir := "./_testgo"
cltest.RunFromDir(t, "", relDir, hostIgnoreByDir[relDir],
cltest.WithExpectedRunErrors(hostExpectedRunErrorByDir[relDir]),
)
}

func TestFilterEmulatorOutput(t *testing.T) {
Expand Down Expand Up @@ -253,7 +300,10 @@ func TestFromTestlibgo(t *testing.T) {
}

func TestRunFromTestlibgo(t *testing.T) {
cltest.RunFromDir(t, "", "./_testlibgo", nil)
relDir := "./_testlibgo"
cltest.RunFromDir(t, "", relDir, hostIgnoreByDir[relDir],
cltest.WithExpectedRunErrors(hostExpectedRunErrorByDir[relDir]),
)
}

func TestFromTestlibc(t *testing.T) {
Expand All @@ -277,14 +327,21 @@ func TestFromTestrt(t *testing.T) {
}

func TestRunFromTestrt(t *testing.T) {
var ignore []string
ignore := []string{
"./_testrt/unreachable", // compiler directive only; compile coverage is enough
}
expectedRunError := []string{
"./_testrt/panic", // panic/exit is expected for this case
}
if runtime.GOOS == "linux" {
ignore = []string{
ignore = append(ignore,
"./_testrt/asmfull", // Output is macOS-specific.
"./_testrt/fprintf", // Linux uses different stderr symbol (no __stderrp).
}
)
}
cltest.RunFromDir(t, "", "./_testrt", ignore)
cltest.RunFromDir(t, "", "./_testrt", ignore,
cltest.WithExpectedRunErrors(expectedRunError),
)
}

func TestFromTestdata(t *testing.T) {
Expand Down
Loading