cc_test.go

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This test uses the Pdeathsig field of syscall.SysProcAttr, so it only works
     6  // on platforms that support that.
     7  
     8  //go:build linux || (freebsd && amd64)
     9  
    10  // sanitizers_test checks the use of Go with sanitizers like msan, asan, etc.
    11  // See https://github.com/google/sanitizers.
    12  package sanitizers_test
    13  
    14  import (
    15  	"bytes"
    16  	"encoding/json"
    17  	"errors"
    18  	"fmt"
    19  	"internal/testenv"
    20  	"os"
    21  	"os/exec"
    22  	"os/user"
    23  	"path/filepath"
    24  	"regexp"
    25  	"strconv"
    26  	"strings"
    27  	"sync"
    28  	"syscall"
    29  	"testing"
    30  	"time"
    31  	"unicode"
    32  )
    33  
    34  var overcommit struct {
    35  	sync.Once
    36  	value int
    37  	err   error
    38  }
    39  
    40  // requireOvercommit skips t if the kernel does not allow overcommit.
    41  func requireOvercommit(t *testing.T) {
    42  	t.Helper()
    43  
    44  	overcommit.Once.Do(func() {
    45  		var out []byte
    46  		out, overcommit.err = os.ReadFile("/proc/sys/vm/overcommit_memory")
    47  		if overcommit.err != nil {
    48  			return
    49  		}
    50  		overcommit.value, overcommit.err = strconv.Atoi(string(bytes.TrimSpace(out)))
    51  	})
    52  
    53  	if overcommit.err != nil {
    54  		t.Skipf("couldn't determine vm.overcommit_memory (%v); assuming no overcommit", overcommit.err)
    55  	}
    56  	if overcommit.value == 2 {
    57  		t.Skip("vm.overcommit_memory=2")
    58  	}
    59  }
    60  
    61  var env struct {
    62  	sync.Once
    63  	m   map[string]string
    64  	err error
    65  }
    66  
    67  // goEnv returns the output of $(go env) as a map.
    68  func goEnv(key string) (string, error) {
    69  	env.Once.Do(func() {
    70  		var out []byte
    71  		out, env.err = exec.Command("go", "env", "-json").Output()
    72  		if env.err != nil {
    73  			return
    74  		}
    75  
    76  		env.m = make(map[string]string)
    77  		env.err = json.Unmarshal(out, &env.m)
    78  	})
    79  	if env.err != nil {
    80  		return "", env.err
    81  	}
    82  
    83  	v, ok := env.m[key]
    84  	if !ok {
    85  		return "", fmt.Errorf("`go env`: no entry for %v", key)
    86  	}
    87  	return v, nil
    88  }
    89  
    90  // replaceEnv sets the key environment variable to value in cmd.
    91  func replaceEnv(cmd *exec.Cmd, key, value string) {
    92  	if cmd.Env == nil {
    93  		cmd.Env = cmd.Environ()
    94  	}
    95  	cmd.Env = append(cmd.Env, key+"="+value)
    96  }
    97  
    98  // appendExperimentEnv appends comma-separated experiments to GOEXPERIMENT.
    99  func appendExperimentEnv(cmd *exec.Cmd, experiments []string) {
   100  	if cmd.Env == nil {
   101  		cmd.Env = cmd.Environ()
   102  	}
   103  	exps := strings.Join(experiments, ",")
   104  	for _, evar := range cmd.Env {
   105  		c := strings.SplitN(evar, "=", 2)
   106  		if c[0] == "GOEXPERIMENT" {
   107  			exps = c[1] + "," + exps
   108  		}
   109  	}
   110  	cmd.Env = append(cmd.Env, "GOEXPERIMENT="+exps)
   111  }
   112  
   113  // mustRun executes t and fails cmd with a well-formatted message if it fails.
   114  func mustRun(t *testing.T, cmd *exec.Cmd) {
   115  	t.Helper()
   116  	out := new(strings.Builder)
   117  	cmd.Stdout = out
   118  	cmd.Stderr = out
   119  
   120  	err := cmd.Start()
   121  	if err != nil {
   122  		t.Fatalf("%v: %v", cmd, err)
   123  	}
   124  
   125  	if deadline, ok := t.Deadline(); ok {
   126  		timeout := time.Until(deadline)
   127  		timeout -= timeout / 10 // Leave 10% headroom for logging and cleanup.
   128  		timer := time.AfterFunc(timeout, func() {
   129  			cmd.Process.Signal(syscall.SIGQUIT)
   130  		})
   131  		defer timer.Stop()
   132  	}
   133  
   134  	if err := cmd.Wait(); err != nil {
   135  		t.Fatalf("%v exited with %v\n%s", cmd, err, out)
   136  	}
   137  }
   138  
   139  // cc returns a cmd that executes `$(go env CC) $(go env GOGCCFLAGS) $args`.
   140  func cc(args ...string) (*exec.Cmd, error) {
   141  	CC, err := goEnv("CC")
   142  	if err != nil {
   143  		return nil, err
   144  	}
   145  
   146  	GOGCCFLAGS, err := goEnv("GOGCCFLAGS")
   147  	if err != nil {
   148  		return nil, err
   149  	}
   150  
   151  	// Split GOGCCFLAGS, respecting quoting.
   152  	//
   153  	// TODO(bcmills): This code also appears in
   154  	// cmd/cgo/internal/testcarchive/carchive_test.go, and perhaps ought to go in
   155  	// src/cmd/dist/test.go as well. Figure out where to put it so that it can be
   156  	// shared.
   157  	var flags []string
   158  	quote := '\000'
   159  	start := 0
   160  	lastSpace := true
   161  	backslash := false
   162  	for i, c := range GOGCCFLAGS {
   163  		if quote == '\000' && unicode.IsSpace(c) {
   164  			if !lastSpace {
   165  				flags = append(flags, GOGCCFLAGS[start:i])
   166  				lastSpace = true
   167  			}
   168  		} else {
   169  			if lastSpace {
   170  				start = i
   171  				lastSpace = false
   172  			}
   173  			if quote == '\000' && !backslash && (c == '"' || c == '\'') {
   174  				quote = c
   175  				backslash = false
   176  			} else if !backslash && quote == c {
   177  				quote = '\000'
   178  			} else if (quote == '\000' || quote == '"') && !backslash && c == '\\' {
   179  				backslash = true
   180  			} else {
   181  				backslash = false
   182  			}
   183  		}
   184  	}
   185  	if !lastSpace {
   186  		flags = append(flags, GOGCCFLAGS[start:])
   187  	}
   188  
   189  	cmd := exec.Command(CC, flags...)
   190  	cmd.Args = append(cmd.Args, args...)
   191  	return cmd, nil
   192  }
   193  
   194  type version struct {
   195  	name         string
   196  	major, minor int
   197  }
   198  
   199  var compiler struct {
   200  	sync.Once
   201  	version
   202  	err error
   203  }
   204  
   205  // compilerVersion detects the version of $(go env CC).
   206  //
   207  // It returns a non-nil error if the compiler matches a known version schema but
   208  // the version could not be parsed, or if $(go env CC) could not be determined.
   209  func compilerVersion() (version, error) {
   210  	compiler.Once.Do(func() {
   211  		compiler.err = func() error {
   212  			compiler.name = "unknown"
   213  
   214  			cmd, err := cc("--version")
   215  			if err != nil {
   216  				return err
   217  			}
   218  			out, err := cmd.Output()
   219  			if err != nil {
   220  				// Compiler does not support "--version" flag: not Clang or GCC.
   221  				return nil
   222  			}
   223  
   224  			var match [][]byte
   225  			if bytes.HasPrefix(out, []byte("gcc")) {
   226  				compiler.name = "gcc"
   227  				cmd, err := cc("-dumpfullversion", "-dumpversion")
   228  				if err != nil {
   229  					return err
   230  				}
   231  				out, err := cmd.Output()
   232  				if err != nil {
   233  					// gcc, but does not support gcc's "-v" flag?!
   234  					return err
   235  				}
   236  				gccRE := regexp.MustCompile(`(\d+)\.(\d+)`)
   237  				match = gccRE.FindSubmatch(out)
   238  			} else {
   239  				clangRE := regexp.MustCompile(`clang version (\d+)\.(\d+)`)
   240  				if match = clangRE.FindSubmatch(out); len(match) > 0 {
   241  					compiler.name = "clang"
   242  				}
   243  			}
   244  
   245  			if len(match) < 3 {
   246  				return nil // "unknown"
   247  			}
   248  			if compiler.major, err = strconv.Atoi(string(match[1])); err != nil {
   249  				return err
   250  			}
   251  			if compiler.minor, err = strconv.Atoi(string(match[2])); err != nil {
   252  				return err
   253  			}
   254  			return nil
   255  		}()
   256  	})
   257  	return compiler.version, compiler.err
   258  }
   259  
   260  // compilerSupportsLocation reports whether the compiler should be
   261  // able to provide file/line information in backtraces.
   262  func compilerSupportsLocation() bool {
   263  	compiler, err := compilerVersion()
   264  	if err != nil {
   265  		return false
   266  	}
   267  	switch compiler.name {
   268  	case "gcc":
   269  		return compiler.major >= 10
   270  	case "clang":
   271  		// TODO(65606): The clang toolchain on the LUCI builders is not built against
   272  		// zlib, the ASAN runtime can't actually symbolize its own stack trace. Once
   273  		// this is resolved, one way or another, switch this back to 'true'. We still
   274  		// have coverage from the 'gcc' case above.
   275  		if inLUCIBuild() {
   276  			return false
   277  		}
   278  		return true
   279  	default:
   280  		return false
   281  	}
   282  }
   283  
   284  // inLUCIBuild returns true if we're currently executing in a LUCI build.
   285  func inLUCIBuild() bool {
   286  	u, err := user.Current()
   287  	if err != nil {
   288  		return false
   289  	}
   290  	return testenv.Builder() != "" && u.Username == "swarming"
   291  }
   292  
   293  // compilerRequiredTsanVersion reports whether the compiler is the version required by Tsan.
   294  // Only restrictions for ppc64le are known; otherwise return true.
   295  func compilerRequiredTsanVersion(goos, goarch string) bool {
   296  	compiler, err := compilerVersion()
   297  	if err != nil {
   298  		return false
   299  	}
   300  	if compiler.name == "gcc" && goarch == "ppc64le" {
   301  		return compiler.major >= 9
   302  	}
   303  	return true
   304  }
   305  
   306  // compilerRequiredAsanVersion reports whether the compiler is the version required by Asan.
   307  func compilerRequiredAsanVersion(goos, goarch string) bool {
   308  	compiler, err := compilerVersion()
   309  	if err != nil {
   310  		return false
   311  	}
   312  	switch compiler.name {
   313  	case "gcc":
   314  		if goarch == "loong64" {
   315  			return compiler.major >= 14
   316  		}
   317  		if goarch == "ppc64le" {
   318  			return compiler.major >= 9
   319  		}
   320  		return compiler.major >= 7
   321  	case "clang":
   322  		if goarch == "loong64" {
   323  			return compiler.major >= 16
   324  		}
   325  		return compiler.major >= 9
   326  	default:
   327  		return false
   328  	}
   329  }
   330  
   331  type compilerCheck struct {
   332  	once sync.Once
   333  	err  error
   334  	skip bool // If true, skip with err instead of failing with it.
   335  }
   336  
   337  type config struct {
   338  	sanitizer string
   339  
   340  	cFlags, ldFlags, goFlags []string
   341  
   342  	sanitizerCheck, runtimeCheck compilerCheck
   343  }
   344  
   345  var configs struct {
   346  	sync.Mutex
   347  	m map[string]*config
   348  }
   349  
   350  // configure returns the configuration for the given sanitizer.
   351  func configure(sanitizer string) *config {
   352  	configs.Lock()
   353  	defer configs.Unlock()
   354  	if c, ok := configs.m[sanitizer]; ok {
   355  		return c
   356  	}
   357  
   358  	c := &config{
   359  		sanitizer: sanitizer,
   360  		cFlags:    []string{"-fsanitize=" + sanitizer},
   361  		ldFlags:   []string{"-fsanitize=" + sanitizer},
   362  	}
   363  
   364  	if testing.Verbose() {
   365  		c.goFlags = append(c.goFlags, "-x")
   366  	}
   367  
   368  	switch sanitizer {
   369  	case "memory":
   370  		c.goFlags = append(c.goFlags, "-msan")
   371  
   372  	case "thread":
   373  		c.goFlags = append(c.goFlags, "--installsuffix=tsan")
   374  		compiler, _ := compilerVersion()
   375  		if compiler.name == "gcc" {
   376  			c.cFlags = append(c.cFlags, "-fPIC")
   377  			c.ldFlags = append(c.ldFlags, "-fPIC", "-static-libtsan")
   378  		}
   379  
   380  	case "address":
   381  		c.goFlags = append(c.goFlags, "-asan")
   382  		// Set the debug mode to print the C stack trace.
   383  		c.cFlags = append(c.cFlags, "-g")
   384  
   385  	case "fuzzer":
   386  		c.goFlags = append(c.goFlags, "-tags=libfuzzer", "-gcflags=-d=libfuzzer")
   387  
   388  	default:
   389  		panic(fmt.Sprintf("unrecognized sanitizer: %q", sanitizer))
   390  	}
   391  
   392  	if configs.m == nil {
   393  		configs.m = make(map[string]*config)
   394  	}
   395  	configs.m[sanitizer] = c
   396  	return c
   397  }
   398  
   399  // goCmd returns a Cmd that executes "go $subcommand $args" with appropriate
   400  // additional flags and environment.
   401  func (c *config) goCmd(subcommand string, args ...string) *exec.Cmd {
   402  	return c.goCmdWithExperiments(subcommand, args, nil)
   403  }
   404  
   405  // goCmdWithExperiments returns a Cmd that executes
   406  // "GOEXPERIMENT=$experiments go $subcommand $args" with appropriate
   407  // additional flags and CGO-related environment variables.
   408  func (c *config) goCmdWithExperiments(subcommand string, args []string, experiments []string) *exec.Cmd {
   409  	cmd := exec.Command("go", subcommand)
   410  	cmd.Args = append(cmd.Args, c.goFlags...)
   411  	cmd.Args = append(cmd.Args, args...)
   412  	replaceEnv(cmd, "CGO_CFLAGS", strings.Join(c.cFlags, " "))
   413  	replaceEnv(cmd, "CGO_LDFLAGS", strings.Join(c.ldFlags, " "))
   414  	appendExperimentEnv(cmd, experiments)
   415  	return cmd
   416  }
   417  
   418  // skipIfCSanitizerBroken skips t if the C compiler does not produce working
   419  // binaries as configured.
   420  func (c *config) skipIfCSanitizerBroken(t *testing.T) {
   421  	check := &c.sanitizerCheck
   422  	check.once.Do(func() {
   423  		check.skip, check.err = c.checkCSanitizer()
   424  	})
   425  	if check.err != nil {
   426  		t.Helper()
   427  		if check.skip {
   428  			t.Skip(check.err)
   429  		}
   430  		t.Fatal(check.err)
   431  	}
   432  }
   433  
   434  var cMain = []byte(`
   435  int main() {
   436  	return 0;
   437  }
   438  `)
   439  
   440  var cLibFuzzerInput = []byte(`
   441  #include <stddef.h>
   442  int LLVMFuzzerTestOneInput(char *data, size_t size) {
   443  	return 0;
   444  }
   445  `)
   446  
   447  func (c *config) checkCSanitizer() (skip bool, err error) {
   448  	dir, err := os.MkdirTemp("", c.sanitizer)
   449  	if err != nil {
   450  		return false, fmt.Errorf("failed to create temp directory: %v", err)
   451  	}
   452  	defer os.RemoveAll(dir)
   453  
   454  	src := filepath.Join(dir, "return0.c")
   455  	cInput := cMain
   456  	if c.sanitizer == "fuzzer" {
   457  		// libFuzzer generates the main function itself, and uses a different input.
   458  		cInput = cLibFuzzerInput
   459  	}
   460  	if err := os.WriteFile(src, cInput, 0600); err != nil {
   461  		return false, fmt.Errorf("failed to write C source file: %v", err)
   462  	}
   463  
   464  	dst := filepath.Join(dir, "return0")
   465  	cmd, err := cc(c.cFlags...)
   466  	if err != nil {
   467  		return false, err
   468  	}
   469  	cmd.Args = append(cmd.Args, c.ldFlags...)
   470  	cmd.Args = append(cmd.Args, "-o", dst, src)
   471  	out, err := cmd.CombinedOutput()
   472  	if err != nil {
   473  		if bytes.Contains(out, []byte("-fsanitize")) &&
   474  			(bytes.Contains(out, []byte("unrecognized")) ||
   475  				bytes.Contains(out, []byte("unsupported"))) {
   476  			return true, errors.New(string(out))
   477  		}
   478  		return true, fmt.Errorf("%#q failed: %v\n%s", strings.Join(cmd.Args, " "), err, out)
   479  	}
   480  
   481  	if c.sanitizer == "fuzzer" {
   482  		// For fuzzer, don't try running the test binary. It never finishes.
   483  		return false, nil
   484  	}
   485  
   486  	if out, err := exec.Command(dst).CombinedOutput(); err != nil {
   487  		if os.IsNotExist(err) {
   488  			return true, fmt.Errorf("%#q failed to produce executable: %v", strings.Join(cmd.Args, " "), err)
   489  		}
   490  		snippet, _, _ := bytes.Cut(out, []byte("\n"))
   491  		return true, fmt.Errorf("%#q generated broken executable: %v\n%s", strings.Join(cmd.Args, " "), err, snippet)
   492  	}
   493  
   494  	return false, nil
   495  }
   496  
   497  // skipIfRuntimeIncompatible skips t if the Go runtime is suspected not to work
   498  // with cgo as configured.
   499  func (c *config) skipIfRuntimeIncompatible(t *testing.T) {
   500  	check := &c.runtimeCheck
   501  	check.once.Do(func() {
   502  		check.skip, check.err = c.checkRuntime()
   503  	})
   504  	if check.err != nil {
   505  		t.Helper()
   506  		if check.skip {
   507  			t.Skip(check.err)
   508  		}
   509  		t.Fatal(check.err)
   510  	}
   511  }
   512  
   513  func (c *config) checkRuntime() (skip bool, err error) {
   514  	if c.sanitizer != "thread" {
   515  		return false, nil
   516  	}
   517  
   518  	// libcgo.h sets CGO_TSAN if it detects TSAN support in the C compiler.
   519  	// Dump the preprocessor defines to check that works.
   520  	// (Sometimes it doesn't: see https://golang.org/issue/15983.)
   521  	cmd, err := cc(c.cFlags...)
   522  	if err != nil {
   523  		return false, err
   524  	}
   525  	cmd.Args = append(cmd.Args, "-dM", "-E", "../../../../runtime/cgo/libcgo.h")
   526  	cmdStr := strings.Join(cmd.Args, " ")
   527  	out, err := cmd.CombinedOutput()
   528  	if err != nil {
   529  		return false, fmt.Errorf("%#q exited with %v\n%s", cmdStr, err, out)
   530  	}
   531  	if !bytes.Contains(out, []byte("#define CGO_TSAN")) {
   532  		return true, fmt.Errorf("%#q did not define CGO_TSAN", cmdStr)
   533  	}
   534  	return false, nil
   535  }
   536  
   537  // srcPath returns the path to the given file relative to this test's source tree.
   538  func srcPath(path string) string {
   539  	return filepath.Join("testdata", path)
   540  }
   541  
   542  // A tempDir manages a temporary directory within a test.
   543  type tempDir struct {
   544  	base string
   545  }
   546  
   547  func (d *tempDir) RemoveAll(t *testing.T) {
   548  	t.Helper()
   549  	if d.base == "" {
   550  		return
   551  	}
   552  	if err := os.RemoveAll(d.base); err != nil {
   553  		t.Fatalf("Failed to remove temp dir: %v", err)
   554  	}
   555  }
   556  
   557  func (d *tempDir) Base() string {
   558  	return d.base
   559  }
   560  
   561  func (d *tempDir) Join(name string) string {
   562  	return filepath.Join(d.base, name)
   563  }
   564  
   565  func newTempDir(t *testing.T) *tempDir {
   566  	t.Helper()
   567  	dir, err := os.MkdirTemp("", filepath.Dir(t.Name()))
   568  	if err != nil {
   569  		t.Fatalf("Failed to create temp dir: %v", err)
   570  	}
   571  	return &tempDir{base: dir}
   572  }
   573  
   574  // hangProneCmd returns an exec.Cmd for a command that is likely to hang.
   575  //
   576  // If one of these tests hangs, the caller is likely to kill the test process
   577  // using SIGINT, which will be sent to all of the processes in the test's group.
   578  // Unfortunately, TSAN in particular is prone to dropping signals, so the SIGINT
   579  // may terminate the test binary but leave the subprocess running. hangProneCmd
   580  // configures subprocess to receive SIGKILL instead to ensure that it won't
   581  // leak.
   582  func hangProneCmd(name string, arg ...string) *exec.Cmd {
   583  	cmd := exec.Command(name, arg...)
   584  	cmd.SysProcAttr = &syscall.SysProcAttr{
   585  		Pdeathsig: syscall.SIGKILL,
   586  	}
   587  	return cmd
   588  }
   589
View as plain text