       1 #
       2 # Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
       3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
       4 #
       5 # This code is free software; you can redistribute it and/or modify it
       6 # under the terms of the GNU General Public License version 2 only, as
       7 # published by the Free Software Foundation.  Oracle designates this
       8 # particular file as subject to the "Classpath" exception as provided
       9 # by Oracle in the LICENSE file that accompanied this code.
      10 #
      11 # This code is distributed in the hope that it will be useful, but WITHOUT
      12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
      13 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      14 # version 2 for more details (a copy is included in the LICENSE file that
      15 # accompanied this code).
      16 #
      17 # You should have received a copy of the GNU General Public License version
      18 # 2 along with this work; if not, write to the Free Software Foundation,
      19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
      20 #
      21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
      22 # or visit www.oracle.com if you need additional information or have any
      23 # questions.
      24 #
      25
      26 default: all
      27
      28 include $(SPEC)
      29 include MakeBase.gmk
      30 include FindTests.gmk
      31
      32 # We will always run multiple tests serially
      33 .NOTPARALLEL:
      34
      35 ################################################################################
      36 # Parse global control variables
      37 ################################################################################
      38
      39 ifneq ($(TEST_VM_OPTS), )
      40   ifneq ($(TEST_OPTS), )
      41     TEST_OPTS := $(TEST_OPTS);VM_OPTIONS=$(TEST_VM_OPTS)
      42   else
      43     TEST_OPTS := VM_OPTIONS=$(TEST_VM_OPTS)
      44   endif
      45 endif
      46
      47 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
      48     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR JCOV JCOV_DIFF_CHANGESET, \
      49     STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS, \
      50 ))
      51
      52 # Helper function to propagate TEST_OPTS values.
      53 #
      54 # Note: No spaces are allowed around the arguments.
      55 # Arg $1 The variable in TEST_OPTS to propagate
      56 # Arg $2 The control variable to propagate it to
      57 define SetTestOpt
      58   ifneq ($$(TEST_OPTS_$1), )
      59     $2_$1 := $$(TEST_OPTS_$1)
      60   endif
      61 endef
      62
      63 # Setup _NT_SYMBOL_PATH on Windows, which points to our pdb files.
      64 ifeq ($(call isTargetOs, windows), true)
      65   ifndef _NT_SYMBOL_PATH
      66     SYMBOL_PATH := $(call PathList, $(sort $(patsubst %/, %, $(dir $(wildcard \
      67         $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb))))))
      68     export _NT_SYMBOL_PATH := $(subst \\,\, $(call FixPath, \
      69         $(subst $(DQUOTE),, $(SYMBOL_PATH))))
      70     $(call LogDebug, Setting _NT_SYMBOL_PATH to $(_NT_SYMBOL_PATH))
      71   endif
      72 endif
      73
      74 ################################################################################
      75 # Hook to include the corresponding custom file, if present.
      76 $(eval $(call IncludeCustomExtension, RunTests.gmk))
      77 ################################################################################
      78
      79 # This is the JDK that we will test
      80 JDK_UNDER_TEST := $(JDK_IMAGE_DIR)
      81
      82 TEST_RESULTS_DIR := $(OUTPUTDIR)/test-results
      83 TEST_SUPPORT_DIR := $(OUTPUTDIR)/test-support
      84 TEST_SUMMARY := $(TEST_RESULTS_DIR)/test-summary.txt
      85 TEST_LAST_IDS := $(TEST_SUPPORT_DIR)/test-last-ids.txt
      86
      87 ifeq ($(CUSTOM_ROOT), )
      88   JTREG_TOPDIR := $(TOPDIR)
      89 else
      90   JTREG_TOPDIR := $(CUSTOM_ROOT)
      91 endif
      92
      93 JTREG_FAILURE_HANDLER_DIR := $(TEST_IMAGE_DIR)/failure_handler
      94 JTREG_FAILURE_HANDLER := $(JTREG_FAILURE_HANDLER_DIR)/jtregFailureHandler.jar
      95
      96 JTREG_FAILURE_HANDLER_TIMEOUT ?= 0
      97
      98 ifneq ($(wildcard $(JTREG_FAILURE_HANDLER)), )
      99   JTREG_FAILURE_HANDLER_OPTIONS := \
     100       -timeoutHandlerDir:$(JTREG_FAILURE_HANDLER) \
     101       -observerDir:$(JTREG_FAILURE_HANDLER) \
     102       -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \
     103       -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \
     104       -timeoutHandlerTimeout:$(JTREG_FAILURE_HANDLER_TIMEOUT)
     105 endif
     106
     107 GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, \
     108     $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
     109 GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, \
     110     $(GTEST_LAUNCHER_DIRS)))
     111
     112 COV_ENVIRONMENT :=
     113 JTREG_COV_OPTIONS :=
     114
     115 ifeq ($(TEST_OPTS_JCOV), true)
     116   JCOV_OUTPUT_DIR := $(TEST_RESULTS_DIR)/jcov-output
     117   JCOV_GRABBER_LOG := $(JCOV_OUTPUT_DIR)/grabber.log
     118   JCOV_RESULT_FILE := $(JCOV_OUTPUT_DIR)/result.xml
     119   JCOV_REPORT := $(JCOV_OUTPUT_DIR)/report
     120   JCOV_MEM_OPTIONS := -Xms64m -Xmx4g
     121
     122   # Replace our normal test JDK with the JCov image.
     123   JDK_UNDER_TEST := $(JCOV_IMAGE_DIR)
     124
     125   COV_ENVIRONMENT += JAVA_TOOL_OPTIONS="$(JCOV_MEM_OPTIONS)" \
     126       _JAVA_OPTIONS="$(JCOV_MEM_OPTIONS)"
     127   JTREG_COV_OPTIONS += -e:JAVA_TOOL_OPTIONS='$(JCOV_MEM_OPTIONS)' \
     128       -e:_JAVA_OPTIONS='$(JCOV_MEM_OPTIONS)'
     129 endif
     130
     131 ifeq ($(GCOV_ENABLED), true)
     132   GCOV_OUTPUT_DIR := $(TEST_RESULTS_DIR)/gcov-output
     133   COV_ENVIRONMENT += GCOV_PREFIX="$(GCOV_OUTPUT_DIR)"
     134   JTREG_COV_OPTIONS += -e:GCOV_PREFIX="$(GCOV_OUTPUT_DIR)"
     135 endif
     136
     137 ################################################################################
     138 # Setup global test running parameters
     139 ################################################################################
     140
     141 # Each factor variable comes in 3 variants. The first one is reserved for users
     142 # to use on command line. The other two are for predifined configurations in JDL
     143 # and for machine specific configurations respectively.
     144 TEST_JOBS_FACTOR ?= 1
     145 TEST_JOBS_FACTOR_JDL ?= 1
     146 TEST_JOBS_FACTOR_MACHINE ?= 1
     147
     148 ifeq ($(TEST_JOBS), 0)
     149   CORES_DIVIDER := 2
     150   # For some big multi-core machines with low ulimit -u setting we hit the max
     151   # threads/process limit. In such a setup the memory/cores-only-guided
     152   # TEST_JOBS config is insufficient. From experience a concurrency setting of
     153   # 14 works reasonably well for low ulimit values (<= 4096). Thus, use
     154   # divider 4096/14. For high ulimit -u values this shouldn't make a difference.
     155   ULIMIT_DIVIDER := (4096/14)
     156   PROC_ULIMIT := -1
     157   ifneq ($(OPENJDK_TARGET_OS), windows)
     158     PROC_ULIMIT := $(shell $(ULIMIT) -u)
     159     ifeq ($(PROC_ULIMIT), unlimited)
     160       PROC_ULIMIT := -1
     161     endif
     162   endif
     163   MEMORY_DIVIDER := 2048
     164   TEST_JOBS := $(shell $(AWK) \
     165     'BEGIN { \
     166       c = $(NUM_CORES) / $(CORES_DIVIDER); \
     167       m = $(MEMORY_SIZE) / $(MEMORY_DIVIDER); \
     168       u = $(PROC_ULIMIT); \
     169       if (u > -1) { \
     170         u = u / $(ULIMIT_DIVIDER); \
     171         if (u < c) c = u; \
     172       } \
     173       if (c > m) c = m; \
     174       c = c * $(TEST_JOBS_FACTOR); \
     175       c = c * $(TEST_JOBS_FACTOR_JDL); \
     176       c = c * $(TEST_JOBS_FACTOR_MACHINE); \
     177       if (c < 1) c = 1; \
     178       printf "%.0f", c; \
     179     }')
     180 endif
     181
     182 ################################################################################
     183 # Parse control variables
     184 ################################################################################
     185
     186 ifneq ($(TEST_OPTS), )
     187   # Inform the user
     188   $(info Running tests using TEST_OPTS control variable '$(TEST_OPTS)')
     189 endif
     190
     191 ### Jtreg
     192
     193 $(eval $(call SetTestOpt,VM_OPTIONS,JTREG))
     194 $(eval $(call SetTestOpt,JAVA_OPTIONS,JTREG))
     195
     196 $(eval $(call SetTestOpt,JOBS,JTREG))
     197 $(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
     198 $(eval $(call SetTestOpt,FAILURE_HANDLER_TIMEOUT,JTREG))
     199
     200 $(eval $(call ParseKeywordVariable, JTREG, \
     201     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR FAILURE_HANDLER_TIMEOUT \
     202         TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM RUN_PROBLEM_LISTS \
     203         RETRY_COUNT MAX_OUTPUT, \
     204     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS KEYWORDS \
     205         EXTRA_PROBLEM_LISTS LAUNCHER_OPTIONS, \
     206 ))
     207
     208 ifneq ($(JTREG), )
     209   # Inform the user
     210   $(info Running tests using JTREG control variable '$(JTREG)')
     211 endif
     212
     213 ### Gtest
     214
     215 $(eval $(call SetTestOpt,VM_OPTIONS,GTEST))
     216 $(eval $(call SetTestOpt,JAVA_OPTIONS,GTEST))
     217
     218 $(eval $(call ParseKeywordVariable, GTEST, \
     219     SINGLE_KEYWORDS := REPEAT, \
     220     STRING_KEYWORDS := OPTIONS VM_OPTIONS JAVA_OPTIONS, \
     221 ))
     222
     223 ifneq ($(GTEST), )
     224   # Inform the user
     225   $(info Running tests using GTEST control variable '$(GTEST)')
     226 endif
     227
     228 ### Microbenchmarks
     229
     230 $(eval $(call SetTestOpt,VM_OPTIONS,MICRO))
     231 $(eval $(call SetTestOpt,JAVA_OPTIONS,MICRO))
     232
     233 $(eval $(call ParseKeywordVariable, MICRO, \
     234     SINGLE_KEYWORDS := ITER FORK TIME WARMUP_ITER WARMUP_TIME, \
     235     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS RESULTS_FORMAT TEST_JDK \
     236         BENCHMARKS_JAR, \
     237 ))
     238
     239 ifneq ($(MICRO), )
     240   # Inform the user
     241   $(info Running tests using MICRO control variable '$(MICRO)')
     242 endif
     243
     244
     245 ################################################################################
     246 # Component-specific Jtreg settings
     247 ################################################################################
     248
     249 hotspot_JTREG_MAX_MEM := 0
     250 hotspot_JTREG_ASSERT := false
     251 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
     252 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
     253 lib-test_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/lib-test/jtreg/native
     254
     255 jdk_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jdk/ProblemList.txt
     256 jaxp_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jaxp/ProblemList.txt
     257 langtools_JTREG_PROBLEM_LIST += $(TOPDIR)/test/langtools/ProblemList.txt
     258 hotspot_JTREG_PROBLEM_LIST += $(TOPDIR)/test/hotspot/jtreg/ProblemList.txt
     259 lib-test_JTREG_PROBLEM_LIST += $(TOPDIR)/test/lib-test/ProblemList.txt
     260
     261 ################################################################################
     262 # Parse test selection
     263 #
     264 # The user has given a test selection in the TEST variable. We must parse it
     265 # and determine what that means in terms of actual calls to the test framework.
     266 #
     267 # The parse functions take as argument a test specification as given by the
     268 # user, and returns a fully qualified test descriptor if it was a match, or
     269 # nothing if not. A single test specification can result in multiple test
     270 # descriptors being returned. A valid test descriptor must always be accepted
     271 # and returned identically.
     272 ################################################################################
     273
     274 # Helper function to determine if a test specification is a Gtest test
     275 #
     276 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
     277 # test filter string, and an optional "/<variant>" to select a specific JVM
     278 # variant. If no variant is specified, all found variants are tested.
     279 define ParseGtestTestSelection
     280   $(if $(filter gtest%, $1), \
     281     $(if $(filter gtest, $1), \
     282       $(addprefix gtest:all/, $(GTEST_VARIANTS)) \
     283     , \
     284       $(if $(strip $(or $(filter gtest/%, $1) $(filter gtest:/%, $1))), \
     285         $(patsubst gtest:/%, gtest:all/%, $(patsubst gtest/%, gtest:/%, $1)) \
     286       , \
     287         $(if $(filter gtest:%, $1), \
     288           $(if $(findstring /, $1), \
     289             $1 \
     290           , \
     291             $(addprefix $1/, $(GTEST_VARIANTS)) \
     292           ) \
     293         ) \
     294       ) \
     295     ) \
     296   )
     297 endef
     298
     299 # Helper function to determine if a test specification is a microbenchmark test
     300 #
     301 # It is a microbenchmark test if it is either "micro", or "micro:" followed by
     302 # an optional test filter string.
     303 define ParseMicroTestSelection
     304   $(if $(filter micro%, $1), \
     305     $(if $(filter micro, $1), \
     306       micro:all \
     307     , \
     308       $(if $(filter micro:, $1), \
     309         micro:all \
     310       , \
     311         $1 \
     312       ) \
     313     ) \
     314   )
     315 endef
     316
     317 # Helper function that removes the TOPDIR part
     318 CleanupJtregPath = \
     319   $(strip $(patsubst %/, %, $(subst $(JTREG_TOPDIR)/,, $1)))
     320
     321 # Take a partial Jtreg root path and return a full, absolute path to that Jtreg
     322 # root. Also support having "hotspot" as an alias for "hotspot/jtreg".
     323 ExpandJtregRoot = \
     324   $(call CleanupJtregPath, $(wildcard \
     325     $(if $(filter /%, $1), \
     326       $(if $(wildcard $(strip $1)/TEST.ROOT), \
     327         $1 \
     328       ) \
     329     , \
     330       $(filter $(addprefix %, $1), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \
     331       $(filter $(addprefix %, $(strip $1)/jtreg), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \
     332     ) \
     333   ))
     334
     335 # Take a partial Jtreg test path and return a full, absolute path to that Jtreg
     336 # test. Also support having "hotspot" as an alias for "hotspot/jtreg".
     337 ExpandJtregPath = \
     338   $(if $(call ExpandJtregRoot, $1), \
     339     $(call ExpandJtregRoot, $1) \
     340   , \
     341     $(call CleanupJtregPath, $(wildcard \
     342       $(if $(filter /%, $1), \
     343         $1 \
     344       , \
     345         $(addsuffix /$(strip $1), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \
     346         $(addsuffix $(strip $(patsubst hotspot/%, /hotspot/jtreg/%, $1)), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \
     347       ) \
     348     )) \
     349   )
     350
     351 # Helper function to determine if a test specification is a Jtreg test
     352 #
     353 # It is a Jtreg test if it optionally begins with jtreg:, and then is either
     354 # an unspecified group name (possibly prefixed by :), or a group in a
     355 # specified test root, or a path to a test or test directory,
     356 # either absolute or relative to any of the TEST_BASEDIRS or test roots.
     357 define ParseJtregTestSelection
     358   $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \
     359   $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \
     360     $(eval TEST_NAME := :$(TEST_NAME)) \
     361   ) \
     362   $(if $(findstring :, $(TEST_NAME)), \
     363     $(if $(filter :%, $(TEST_NAME)), \
     364       $(eval TEST_GROUP := $(patsubst :%, %, $(TEST_NAME))) \
     365       $(eval TEST_ROOTS := $(foreach test_root, $(JTREG_TESTROOTS), \
     366           $(call CleanupJtregPath, $(test_root)))) \
     367     , \
     368       $(eval TEST_PATH := $(word 1, $(subst :, $(SPACE), $(TEST_NAME)))) \
     369       $(eval TEST_GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \
     370       $(eval TEST_ROOTS := $(call ExpandJtregRoot, $(TEST_PATH))) \
     371     ) \
     372     $(foreach test_root, $(TEST_ROOTS), \
     373       $(if $(filter /%, $(test_root)), \
     374         jtreg:$(test_root):$(TEST_GROUP) \
     375       , \
     376         $(if $(filter $(TEST_GROUP), $($(JTREG_TOPDIR)/$(test_root)_JTREG_TEST_GROUPS)), \
     377           jtreg:$(test_root):$(TEST_GROUP) \
     378         ) \
     379       ) \
     380     ) \
     381   , \
     382     $(eval TEST_PATHS := $(call ExpandJtregPath, $(TEST_NAME))) \
     383     $(foreach test_path, $(TEST_PATHS), \
     384       jtreg:$(test_path) \
     385     ) \
     386   )
     387 endef
     388
     389 # Helper function to determine if a test specification is a special test
     390 #
     391 # It is a special test if it is "special:" followed by a test name,
     392 # if it is "make:" or "make-" followed by a make test, or any of the special
     393 # test names as a single word.
     394 define ParseSpecialTestSelection
     395   $(if $(filter special:%, $1), \
     396     $1 \
     397   ) \
     398   $(if $(filter make%, $1), \
     399     $(if $(filter make:%, $1), \
     400       special:$(strip $1) \
     401     ) \
     402     $(if $(filter make-%, $1), \
     403       special:$(patsubst make-%,make:%, $1) \
     404     ) \
     405     $(if $(filter make, $1), \
     406       special:make:all \
     407     )
     408   ) \
     409   $(if $(filter failure-handler, $1), \
     410     special:$(strip $1) \
     411   )
     412 endef
     413
     414 ifeq ($(TEST), )
     415   $(info No test selection given in TEST!)
     416   $(info Please use e.g. 'make test TEST=tier1' or 'make test-tier1')
     417   $(info See doc/testing.[md|html] for help)
     418   $(error Cannot continue)
     419 endif
     420
     421 # Now intelligently convert the test selection given by the user in TEST
     422 # into a list of fully qualified test descriptors of the tests to run.
     423 TESTS_TO_RUN :=
     424 $(foreach test, $(TEST), \
     425   $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \
     426   $(if $(strip $(PARSED_TESTS)), , \
     427     $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \
     428   ) \
     429   $(if $(strip $(PARSED_TESTS)), , \
     430     $(eval PARSED_TESTS += $(call ParseMicroTestSelection, $(test))) \
     431   ) \
     432   $(if $(strip $(PARSED_TESTS)), , \
     433     $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \
     434   ) \
     435   $(if $(strip $(PARSED_TESTS)), , \
     436     $(eval PARSED_TESTS += $(call ParseSpecialTestSelection, $(test))) \
     437   ) \
     438   $(if $(strip $(PARSED_TESTS)), , \
     439     $(eval UNKNOWN_TEST := $(test)) \
     440   ) \
     441   $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \
     442 )
     443
     444 ifneq ($(UNKNOWN_TEST), )
     445   $(info Unknown test selection: '$(UNKNOWN_TEST)')
     446   $(info See doc/testing.[md|html] for help)
     447   $(error Cannot continue)
     448 endif
     449
     450 TESTS_TO_RUN := $(strip $(TESTS_TO_RUN))
     451
     452
     453 # Present the result of our parsing to the user
     454 $(info Test selection '$(TEST)', will run:)
     455 $(foreach test, $(TESTS_TO_RUN), $(info * $(test)))
     456
     457
     458 ################################################################################
     459 # Functions for setting up rules for running the selected tests
     460 #
     461 # The SetupRun*Test functions all have the same interface:
     462 #
     463 # Parameter 1 is the name of the rule. This is the test id, based on the test
     464 # descriptor, and this is also used as variable prefix, and the targets
     465 # generated are listed in a variable by that name.
     466 #
     467 # Remaining parameters are named arguments. Currently this is only:
     468 #   TEST -- The properly formatted fully qualified test descriptor
     469 #
     470 # After the rule named by the test id has been executed, the following
     471 # variables will be available:
     472 # testid_TOTAL - the total number of tests run
     473 # testid_PASSED - the number of successful tests
     474 # testid_FAILED - the number of failed tests
     475 # testid_ERROR - the number of tests was neither successful or failed
     476 #
     477 ################################################################################
     478
     479 ### Rules for Gtest
     480
     481 SetupRunGtestTest = $(NamedParamsMacroTemplate)
     482 define SetupRunGtestTestBody
     483   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
     484   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
     485   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
     486
     487   $1_VARIANT :=  $$(lastword $$(subst /, , $$($1_TEST)))
     488   ifeq ($$(filter $$($1_VARIANT), $$(GTEST_VARIANTS)), )
     489     $$(error Invalid gtest variant '$$($1_VARIANT)'. Valid variants: $$(GTEST_VARIANTS))
     490   endif
     491   $1_TEST_NAME := $$(strip $$(patsubst %/$$($1_VARIANT), %, \
     492       $$(patsubst gtest:%, %, $$($1_TEST))))
     493   ifneq ($$($1_TEST_NAME), all)
     494     $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
     495   endif
     496
     497   ifneq ($$(GTEST_REPEAT), )
     498     $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
     499   endif
     500
     501   run-test-$1: pre-run-test
     502 	$$(call LogWarn)
     503 	$$(call LogWarn, Running test '$$($1_TEST)')
     504 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
     505 	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, ( \
     506 	    $$(CD) $$($1_TEST_SUPPORT_DIR) && \
     507 	    $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/$$($1_VARIANT)/gtestLauncher \
     508 	        -jdk $(JDK_UNDER_TEST) $$($1_GTEST_FILTER) \
     509 	        --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
     510 	        --gtest_catch_exceptions=0 \
     511 	        $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \
     512 	        $$(GTEST_JAVA_OPTIONS) \
     513 	        > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) \
     514 	    && $$(ECHO) $$$$? > $$($1_EXITCODE) \
     515 	    || $$(ECHO) $$$$? > $$($1_EXITCODE) \
     516 	))
     517
     518   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
     519
     520   parse-test-$1: run-test-$1
     521 	$$(call LogWarn, Finished running test '$$($1_TEST)')
     522 	$$(call LogWarn, Test report is stored in $$(strip \
     523 	    $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
     524 	$$(if $$(wildcard $$($1_RESULT_FILE)), \
     525 	  $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
     526 	      test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE))) \
     527 	  $$(if $$($1_TOTAL), , $$(eval $1_TOTAL := 0)) \
     528 	  $$(eval $1_PASSED := $$(shell $$(AWK) '/\[  PASSED  \] .* tests?./ \
     529 	      { print $$$$4 }' $$($1_RESULT_FILE))) \
     530 	  $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
     531 	  $$(eval $1_FAILED := $$(shell $$(AWK) '/\[  FAILED  \] .* tests?, \
     532 	      listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) \
     533 	  $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
     534 	  $$(eval $1_ERROR := $$(shell \
     535 	      $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED))) \
     536 	, \
     537 	  $$(eval $1_PASSED := 0) \
     538 	  $$(eval $1_FAILED := 0) \
     539 	  $$(eval $1_ERROR := 1) \
     540 	  $$(eval $1_TOTAL := 1) \
     541 	)
     542
     543   $1: run-test-$1 parse-test-$1
     544
     545   TARGETS += $1 run-test-$1 parse-test-$1
     546   TEST_TARGETS += parse-test-$1
     547
     548 endef
     549
     550 ################################################################################
     551
     552 ### Rules for Microbenchmarks
     553
     554 # Helper function for SetupRunMicroTest. Set a MICRO_* variable from, in order:
     555 # 1) Specified by user on command line
     556 # 2) Generic default
     557 #
     558 # Note: No spaces are allowed around the arguments.
     559 # Arg $1 The test ID (i.e. $1 in SetupRunMicroTest)
     560 # Arg $2 Base variable, e.g. MICRO_TEST_JDK
     561 # Arg $3 The default value (optional)
     562 define SetMicroValue
     563   ifneq ($$($2), )
     564     $1_$2 := $$($2)
     565   else
     566     ifneq ($3, )
     567       $1_$2 := $3
     568     endif
     569   endif
     570 endef
     571
     572 SetupRunMicroTest = $(NamedParamsMacroTemplate)
     573 define SetupRunMicroTestBody
     574   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
     575   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
     576   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
     577
     578   $1_TEST_NAME := $$(strip $$(patsubst micro:%, %, $$($1_TEST)))
     579
     580   $$(eval $$(call SetMicroValue,$1,MICRO_BENCHMARKS_JAR,$$(TEST_IMAGE_DIR)/micro/benchmarks.jar))
     581   $$(eval $$(call SetMicroValue,$1,MICRO_TEST_JDK,$$(JDK_UNDER_TEST)))
     582   $$(eval $$(call SetMicroValue,$1,MICRO_JAVA_OPTIONS))
     583
     584   # Current tests needs to open java.io
     585   $1_MICRO_JAVA_OPTIONS += --add-opens=java.base/java.io=ALL-UNNAMED
     586
     587   # Save output as JSON or CSV file
     588   ifneq ($$(MICRO_RESULTS_FORMAT), )
     589     $1_MICRO_BASIC_OPTIONS += -rf $$(MICRO_RESULTS_FORMAT)
     590     $1_MICRO_BASIC_OPTIONS += -rff $$($1_TEST_RESULTS_DIR)/jmh-result.$(MICRO_RESULTS_FORMAT)
     591   endif
     592
     593   # Set library path for native dependencies
     594   $1_JMH_JVM_ARGS := -Djava.library.path=$$(TEST_IMAGE_DIR)/micro/native
     595
     596   ifneq ($$(MICRO_VM_OPTIONS)$$(MICRO_JAVA_OPTIONS), )
     597     $1_JMH_JVM_ARGS += $$(MICRO_VM_OPTIONS) $$(MICRO_JAVA_OPTIONS)
     598   endif
     599
     600   $1_MICRO_VM_OPTIONS := -jvmArgs $(call ShellQuote,$$($1_JMH_JVM_ARGS))
     601
     602   ifneq ($$(MICRO_ITER), )
     603     $1_MICRO_ITER := -i $$(MICRO_ITER)
     604   endif
     605   ifneq ($$(MICRO_FORK), )
     606     $1_MICRO_FORK := -f $$(MICRO_FORK)
     607   endif
     608   ifneq ($$(MICRO_TIME), )
     609     $1_MICRO_TIME := -r $$(MICRO_TIME)
     610   endif
     611   ifneq ($$(MICRO_WARMUP_ITER), )
     612     $1_MICRO_WARMUP_ITER := -wi $$(MICRO_WARMUP_ITER)
     613   endif
     614   ifneq ($$(MICRO_WARMUP_TIME), )
     615     $1_MICRO_WARMUP_TIME := -w $$(MICRO_WARMUP_TIME)
     616   endif
     617
     618   run-test-$1: pre-run-test
     619 	$$(call LogWarn)
     620 	$$(call LogWarn, Running test '$$($1_TEST)')
     621 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
     622 	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/micro, ( \
     623 	    $$(FIXPATH) $$($1_MICRO_TEST_JDK)/bin/java $$($1_MICRO_JAVA_OPTIONS) \
     624 	        -jar $$($1_MICRO_BENCHMARKS_JAR) \
     625 	        $$($1_MICRO_ITER) $$($1_MICRO_FORK) $$($1_MICRO_TIME) \
     626 	        $$($1_MICRO_WARMUP_ITER) $$($1_MICRO_WARMUP_TIME) \
     627 	        $$($1_MICRO_VM_OPTIONS) $$($1_MICRO_BASIC_OPTIONS) $$(MICRO_OPTIONS) \
     628 	        $$($1_TEST_NAME) \
     629 	        > >($(TEE) $$($1_TEST_RESULTS_DIR)/micro.txt) \
     630 	    && $$(ECHO) $$$$? > $$($1_EXITCODE) \
     631 	    || $$(ECHO) $$$$? > $$($1_EXITCODE) \
     632 	))
     633
     634   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/micro.txt
     635
     636   parse-test-$1: run-test-$1
     637 	$$(call LogWarn, Finished running test '$$($1_TEST)')
     638 	$$(call LogWarn, Test report is stored in $$(strip \
     639 	    $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
     640 	$$(if $$(wildcard $$($1_EXITCODE)), \
     641 	  $$(eval $1_EXIT_CODE := $$(shell $$(CAT) $$($1_EXITCODE))) \
     642 	  $$(if $$(filter 0, $$($1_EXIT_CODE)), \
     643 	    $$(eval $1_PASSED := 1) \
     644 	    $$(eval $1_ERROR := 0) \
     645 	  , \
     646 	    $$(eval $1_PASSED := 0) \
     647 	    $$(eval $1_ERROR := 1) \
     648 	  ) \
     649 	  $$(eval $1_FAILED := 0) \
     650 	  $$(eval $1_TOTAL := $$(shell \
     651 	      $$(EXPR) $$($1_PASSED) + $$($1_ERROR))) \
     652 	, \
     653 	  $$(eval $1_PASSED := 0) \
     654 	  $$(eval $1_FAILED := 0) \
     655 	  $$(eval $1_ERROR := 1) \
     656 	  $$(eval $1_TOTAL := 1) \
     657 	)
     658
     659   $1: run-test-$1 parse-test-$1
     660
     661   TARGETS += $1 run-test-$1 parse-test-$1
     662   TEST_TARGETS += parse-test-$1
     663
     664 endef
     665
     666 ################################################################################
     667
     668 ### Rules for Jtreg
     669
     670 # Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order:
     671 # 1) Specified by user on command line
     672 # 2) Component-specific default
     673 # 3) Generic default
     674 #
     675 # Note: No spaces are allowed around the arguments.
     676 # Arg $1 The test ID (i.e. $1 in SetupRunJtregTest)
     677 # Arg $2 Base variable, e.g. JTREG_JOBS
     678 # Arg $3 The default value (optional)
     679 define SetJtregValue
     680   ifneq ($$($2), )
     681     $1_$2 := $$($2)
     682   else
     683     ifneq ($$($$($1_COMPONENT)_$2), )
     684       $1_$2 := $$($$($1_COMPONENT)_$2)
     685     else
     686       ifneq ($3, )
     687         $1_$2 := $3
     688       endif
     689     endif
     690   endif
     691 endef
     692
     693 SetupRunJtregTest = $(NamedParamsMacroTemplate)
     694 define SetupRunJtregTestBody
     695   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
     696   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
     697   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
     698
     699   $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
     700
     701   $1_TEST_ROOT := \
     702       $$(strip $$(foreach root, $$(JTREG_TESTROOTS), \
     703         $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), $$(root)) \
     704       ))
     705   $1_COMPONENT := $$(lastword $$(subst /, $$(SPACE), $$($1_TEST_ROOT)))
     706   # This will work only as long as just hotspot has the additional "jtreg" directory
     707   ifeq ($$($1_COMPONENT), jtreg)
     708     $1_COMPONENT := hotspot
     709   endif
     710
     711   ifeq ($$(JT_HOME), )
     712     $$(info Error: jtreg framework is not found.)
     713     $$(info Please run configure using --with-jtreg.)
     714     $$(error Cannot continue)
     715   endif
     716
     717   # Unfortunately, we need different defaults for some JTREG values,
     718   # depending on what component we're running.
     719
     720   # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
     721   $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
     722   $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
     723   $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,768m))
     724   $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
     725   $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
     726   $$(eval $$(call SetJtregValue,$1,JTREG_PROBLEM_LIST))
     727
     728   # Only the problem list for the current test root should be used.
     729   $1_JTREG_PROBLEM_LIST := $$(filter $$($1_TEST_ROOT)%, $$($1_JTREG_PROBLEM_LIST))
     730
     731   ifneq ($(TEST_JOBS), 0)
     732     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
     733   else
     734     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
     735   endif
     736
     737   # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since
     738   # we may end up with a lot of JVM's
     739   $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $(AWK) 'BEGIN { print 25 / $$($1_JTREG_JOBS); }')
     740
     741   JTREG_TIMEOUT_FACTOR ?= 4
     742
     743   JTREG_VERBOSE ?= fail,error,summary
     744   JTREG_RETAIN ?= fail,error
     745   JTREG_RUN_PROBLEM_LISTS ?= false
     746   JTREG_RETRY_COUNT ?= 0
     747
     748   ifneq ($$(JTREG_LAUNCHER_OPTIONS), )
     749     $1_JTREG_LAUNCHER_OPTIONS += $$(JTREG_LAUNCHER_OPTIONS)
     750   endif
     751
     752   ifneq ($$(JTREG_MAX_OUTPUT), )
     753     $1_JTREG_LAUNCHER_OPTIONS += -Djavatest.maxOutputSize=$$(JTREG_MAX_OUTPUT)
     754   endif
     755
     756   ifneq ($$($1_JTREG_MAX_MEM), 0)
     757     $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
     758     $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
     759   endif
     760
     761   # Make sure the tmp dir is normalized as some tests will react badly otherwise
     762   $1_TEST_TMP_DIR := $$(abspath $$($1_TEST_SUPPORT_DIR)/tmp)
     763
     764   $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
     765       -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
     766       -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT_FACTOR) \
     767       -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE) \
     768       -vmoption:-Djava.io.tmpdir="$$($1_TEST_TMP_DIR)"
     769
     770   $1_JTREG_BASIC_OPTIONS += -automatic -ignore:quiet -Djavatest.security.noSecurityManager=true
     771
     772   # Make it possible to specify the JIB_DATA_DIR for tests using the
     773   # JIB Artifact resolver
     774   $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
     775   # Some tests needs to find a boot JDK using the JDK8_HOME variable.
     776   $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
     777   # If running on Windows, propagate the _NT_SYMBOL_PATH to enable
     778   # symbol lookup in hserr files
     779   ifeq ($$(call isTargetOs, windows), true)
     780     $1_JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH
     781   endif
     782
     783   $1_JTREG_BASIC_OPTIONS += \
     784       $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
     785       $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
     786       #
     787
     788   ifeq ($$($1_JTREG_ASSERT), true)
     789     $1_JTREG_BASIC_OPTIONS += -ea -esa
     790   endif
     791
     792   ifneq ($$($1_JTREG_NATIVEPATH), )
     793     $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
     794   endif
     795
     796   ifeq ($$(JTREG_RUN_PROBLEM_LISTS), true)
     797     JTREG_PROBLEM_LIST_PREFIX := -match:
     798   else
     799     JTREG_PROBLEM_LIST_PREFIX := -exclude:
     800   endif
     801
     802   ifneq ($$($1_JTREG_PROBLEM_LIST), )
     803     $1_JTREG_BASIC_OPTIONS += $$(addprefix $$(JTREG_PROBLEM_LIST_PREFIX), $$($1_JTREG_PROBLEM_LIST))
     804   endif
     805
     806   ifneq ($$(JTREG_EXTRA_PROBLEM_LISTS), )
     807     # Accept both absolute paths as well as relative to the current test root.
     808     $1_JTREG_BASIC_OPTIONS += $$(addprefix $$(JTREG_PROBLEM_LIST_PREFIX), $$(wildcard \
     809         $$(JTREG_EXTRA_PROBLEM_LISTS) \
     810         $$(addprefix $$($1_TEST_ROOT)/, $$(JTREG_EXTRA_PROBLEM_LISTS)) \
     811     ))
     812   endif
     813
     814   ifneq ($$(JIB_HOME), )
     815     $1_JTREG_BASIC_OPTIONS += -e:JIB_HOME=$$(JIB_HOME)
     816   endif
     817
     818   $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_DIR=$(TEST_IMAGE_DIR)
     819
     820   ifneq ($$(JTREG_FAILURE_HANDLER_OPTIONS), )
     821     $1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)"
     822   endif
     823
     824   ifneq ($$(JTREG_KEYWORDS), )
     825     # The keywords string may contain problematic characters and may be quoted
     826     # already when it arrives here. Remove any existing quotes and replace them
     827     # with one set of single quotes.
     828     $1_JTREG_KEYWORDS := \
     829         $$(strip $$(subst $$(SQUOTE),,$$(subst $$(DQUOTE),,$$(JTREG_KEYWORDS))))
     830     ifneq ($$($1_JTREG_KEYWORDS), )
     831       $1_JTREG_BASIC_OPTIONS += -k:'$$($1_JTREG_KEYWORDS)'
     832     endif
     833   endif
     834
     835   clean-workdir-$1:
     836 	$$(RM) -r $$($1_TEST_SUPPORT_DIR)
     837
     838   $1_COMMAND_LINE := \
     839       $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
     840           -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
     841           $$($1_JTREG_BASIC_OPTIONS) \
     842           -testjdk:$$(JDK_UNDER_TEST) \
     843           -dir:$$(JTREG_TOPDIR) \
     844           -reportDir:$$($1_TEST_RESULTS_DIR) \
     845           -workDir:$$($1_TEST_SUPPORT_DIR) \
     846           -status:$$$${JTREG_STATUS} \
     847           $$(JTREG_OPTIONS) \
     848           $$(JTREG_FAILURE_HANDLER_OPTIONS) \
     849           $$(JTREG_COV_OPTIONS) \
     850           $$($1_TEST_NAME) \
     851       && $$(ECHO) $$$$? > $$($1_EXITCODE) \
     852       || $$(ECHO) $$$$? > $$($1_EXITCODE)
     853
     854
     855   ifneq ($$(JTREG_RETRY_COUNT), 0)
     856     $1_COMMAND_LINE := \
     857         for i in {0..$$(JTREG_RETRY_COUNT)}; do \
     858           if [ "$$$$i" != 0 ]; then \
     859             $$(PRINTF) "\nRetrying Jtreg run. Attempt: $$$$i\n"; \
     860           fi; \
     861           $$($1_COMMAND_LINE); \
     862           if [ "`$$(CAT) $$($1_EXITCODE)`" = "0" ]; then \
     863             break; \
     864           fi; \
     865           export JTREG_STATUS="-status:error,fail"; \
     866         done
     867   endif
     868
     869   run-test-$1: pre-run-test clean-workdir-$1
     870 	$$(call LogWarn)
     871 	$$(call LogWarn, Running test '$$($1_TEST)')
     872 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR) \
     873 	    $$($1_TEST_TMP_DIR))
     874 	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, ( \
     875             $$(COV_ENVIRONMENT) $$($1_COMMAND_LINE) \
     876 	))
     877
     878   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
     879
     880   parse-test-$1: run-test-$1
     881 	$$(call LogWarn, Finished running test '$$($1_TEST)')
     882 	$$(call LogWarn, Test report is stored in $$(strip \
     883 	    $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
     884 	$$(if $$(wildcard $$($1_RESULT_FILE)), \
     885 	  $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
     886 	      for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \
     887 	      print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
     888 	  $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
     889 	  $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
     890 	      for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \
     891 	      print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
     892 	  $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
     893 	  $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
     894 	      for (i=1; i<=NF; i++) { if ($$$$i == "error:") \
     895 	      print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
     896 	  $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0)) \
     897 	  $$(eval $1_TOTAL := $$(shell \
     898 	      $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR))) \
     899 	, \
     900 	  $$(eval $1_PASSED := 0) \
     901 	  $$(eval $1_FAILED := 0) \
     902 	  $$(eval $1_ERROR := 1) \
     903 	  $$(eval $1_TOTAL := 1) \
     904 	)
     905
     906   $1: run-test-$1 parse-test-$1 clean-workdir-$1
     907
     908   TARGETS += $1 run-test-$1 parse-test-$1 clean-workdir-$1
     909   TEST_TARGETS += parse-test-$1
     910
     911 endef
     912
     913 ################################################################################
     914
     915 ### Rules for special tests
     916
     917 SetupRunSpecialTest = $(NamedParamsMacroTemplate)
     918 define SetupRunSpecialTestBody
     919   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
     920   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
     921   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
     922
     923   $1_FULL_TEST_NAME := $$(strip $$(patsubst special:%, %, $$($1_TEST)))
     924   ifneq ($$(findstring :, $$($1_FULL_TEST_NAME)), )
     925     $1_TEST_NAME := $$(firstword $$(subst :, ,$$($1_FULL_TEST_NAME)))
     926     $1_TEST_ARGS := $$(strip $$(patsubst special:$$($1_TEST_NAME):%, %, $$($1_TEST)))
     927   else
     928     $1_TEST_NAME := $$($1_FULL_TEST_NAME)
     929     $1_TEST_ARGS :=
     930   endif
     931
     932   ifeq ($$($1_TEST_NAME), failure-handler)
     933     ifeq ($(BUILD_FAILURE_HANDLER), true)
     934       $1_TEST_COMMAND_LINE := \
     935           ($(CD) $(TOPDIR)/make/test && $(MAKE) $(MAKE_ARGS) -f \
     936           BuildFailureHandler.gmk test)
     937     else
     938       $$(error Cannot test failure handler if it is not built)
     939     endif
     940   else ifeq ($$($1_TEST_NAME), make)
     941     $1_TEST_COMMAND_LINE := \
     942         ($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f \
     943         TestMake.gmk $$($1_TEST_ARGS) TEST_SUPPORT_DIR="$$($1_TEST_SUPPORT_DIR)")
     944   else
     945     $$(error Invalid special test specification: $$($1_TEST_NAME))
     946   endif
     947
     948   run-test-$1: pre-run-test
     949 	$$(call LogWarn)
     950 	$$(call LogWarn, Running test '$$($1_TEST)')
     951 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
     952 	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/test-execution, ( \
     953 	    $$($1_TEST_COMMAND_LINE) \
     954 	        > >($(TEE) $$($1_TEST_RESULTS_DIR)/test-output.txt) \
     955 	    && $$(ECHO) $$$$? > $$($1_EXITCODE) \
     956 	    || $$(ECHO) $$$$? > $$($1_EXITCODE) \
     957 	))
     958
     959   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
     960
     961   # We can not parse the various "special" tests.
     962   parse-test-$1: run-test-$1
     963 	$$(call LogWarn, Finished running test '$$($1_TEST)')
     964 	$$(call LogWarn, Test report is stored in $$(strip \
     965 	    $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
     966 	$$(call LogWarn, Warning: Special test results are not properly parsed!)
     967 	$$(eval $1_PASSED := $$(shell \
     968 	  if [ `$(CAT) $$($1_EXITCODE)` = "0" ]; then $(ECHO) 1; else $(ECHO) 0; fi \
     969 	))
     970 	$$(eval $1_FAILED := $$(shell \
     971 	  if [ `$(CAT) $$($1_EXITCODE)` = "0" ]; then $(ECHO) 0; else $(ECHO) 1; fi \
     972 	))
     973 	$$(eval $1_ERROR := 0)
     974 	$$(eval $1_TOTAL := 1)
     975
     976   $1: run-test-$1 parse-test-$1
     977
     978   TARGETS += $1 run-test-$1 parse-test-$1
     979   TEST_TARGETS += parse-test-$1
     980
     981 endef
     982
     983 ################################################################################
     984 # Setup and execute make rules for all selected tests
     985 ################################################################################
     986
     987 # Helper function to determine which handler to use for the given test
     988 UseGtestTestHandler = \
     989   $(if $(filter gtest:%, $1), true)
     990
     991 UseMicroTestHandler = \
     992   $(if $(filter micro:%, $1), true)
     993
     994 UseJtregTestHandler = \
     995   $(if $(filter jtreg:%, $1), true)
     996
     997 UseSpecialTestHandler = \
     998   $(if $(filter special:%, $1), true)
     999
    1000 # Now process each test to run and setup a proper make rule
    1001 $(foreach test, $(TESTS_TO_RUN), \
    1002   $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
    1003       $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \
    1004   $(eval ALL_TEST_IDS += $(TEST_ID)) \
    1005   $(if $(call UseCustomTestHandler, $(test)), \
    1006     $(eval $(call SetupRunCustomTest, $(TEST_ID), \
    1007         TEST := $(test), \
    1008     )) \
    1009   ) \
    1010   $(if $(call UseGtestTestHandler, $(test)), \
    1011     $(eval $(call SetupRunGtestTest, $(TEST_ID), \
    1012         TEST := $(test), \
    1013     )) \
    1014   ) \
    1015   $(if $(call UseMicroTestHandler, $(test)), \
    1016     $(eval $(call SetupRunMicroTest, $(TEST_ID), \
    1017         TEST := $(test), \
    1018     )) \
    1019   ) \
    1020   $(if $(call UseJtregTestHandler, $(test)), \
    1021     $(eval $(call SetupRunJtregTest, $(TEST_ID), \
    1022         TEST := $(test), \
    1023     )) \
    1024   ) \
    1025   $(if $(call UseSpecialTestHandler, $(test)), \
    1026     $(eval $(call SetupRunSpecialTest, $(TEST_ID), \
    1027         TEST := $(test), \
    1028     )) \
    1029   ) \
    1030 )
    1031
    1032 # Sort also removes duplicates, so if there is any we'll get fewer words.
    1033 ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS))))
    1034   $(error Duplicate test specification)
    1035 endif
    1036
    1037
    1038 ################################################################################
    1039 # The main target for RunTests.gmk
    1040 ################################################################################
    1041
    1042 #
    1043 # Provide hooks for adding functionality before and after all tests are run.
    1044 #
    1045
    1046 $(call LogInfo, RunTest setup starting)
    1047
    1048 # This target depends on all actual test having been run (TEST_TARGETS has beeen
    1049 # populated by the SetupRun*Test functions). If you need to provide a teardown
    1050 # hook, you must let it depend on this target.
    1051 run-all-tests: $(TEST_TARGETS)
    1052 	$(call LogInfo, RunTest teardown starting)
    1053
    1054 # This is an abstract target that will be run before any actual tests. Add your
    1055 # target as a dependency to thisif you need "setup" type functionality executed
    1056 # before all tests.
    1057 pre-run-test:
    1058 	$(call LogInfo, RunTest setup done)
    1059
    1060 # This is an abstract target that will be run after all actual tests, but before
    1061 # the test summary. If you need "teardown" type functionality, add your target
    1062 # as a dependency on this, and let the teardown target depend on run-all-tests.
    1063 post-run-test: run-all-tests
    1064 	$(call LogInfo, RunTest teardown done)
    1065
    1066 #
    1067 # Create and print a table of the result of all tests run
    1068 #
    1069 TEST_FAILURE := false
    1070
    1071 run-test-report: post-run-test
    1072 	$(RM) $(TEST_SUMMARY).old 2> /dev/null
    1073 	$(MV) $(TEST_SUMMARY) $(TEST_SUMMARY).old 2> /dev/null || true
    1074 	$(RM) $(TEST_LAST_IDS).old 2> /dev/null
    1075 	$(MV) $(TEST_LAST_IDS) $(TEST_LAST_IDS).old 2> /dev/null || true
    1076 	$(ECHO) >> $(TEST_SUMMARY) ==============================
    1077 	$(ECHO) >> $(TEST_SUMMARY) Test summary
    1078 	$(ECHO) >> $(TEST_SUMMARY) ==============================
    1079 	$(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5s %5s %5s %5s %2s\n" "  " \
    1080 	    TEST TOTAL PASS FAIL ERROR " "
    1081 	$(foreach test, $(TESTS_TO_RUN), \
    1082 	  $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
    1083 	      $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \
    1084 	    $(ECHO) >> $(TEST_LAST_IDS) $(TEST_ID) $(NEWLINE) \
    1085 	  $(eval NAME_PATTERN := $(shell $(ECHO) $(test) | $(TR) -c '\n' '[_*1000]')) \
    1086 	  $(if $(filter __________________________________________________%, $(NAME_PATTERN)), \
    1087 	    $(eval TEST_NAME := ) \
    1088 	    $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s\n" "  " "$(test)"  $(NEWLINE) \
    1089 	  , \
    1090 	    $(eval TEST_NAME := $(test)) \
    1091 	  ) \
    1092 	  $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \
    1093 	    $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
    1094 	        "  " "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
    1095 	        $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "  " $(NEWLINE) \
    1096 	  , \
    1097 	    $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
    1098 	         ">>" "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
    1099 	        $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \
    1100 	    $(eval TEST_FAILURE := true) \
    1101 	  ) \
    1102 	)
    1103 	$(ECHO) >> $(TEST_SUMMARY) ==============================
    1104 	$(if $(filter true, $(TEST_FAILURE)), \
    1105 	  $(ECHO) >> $(TEST_SUMMARY) TEST FAILURE $(NEWLINE) \
    1106 	  $(MKDIR) -p $(MAKESUPPORT_OUTPUTDIR) $(NEWLINE) \
    1107 	  $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \
    1108 	, \
    1109 	  $(ECHO) >> $(TEST_SUMMARY) TEST SUCCESS \
    1110 	)
    1111 	$(ECHO)
    1112 	$(CAT) $(TEST_SUMMARY)
    1113 	$(ECHO)
    1114
    1115 # The main run-test target
    1116 run-test: run-test-report
    1117
    1118 TARGETS += run-all-tests pre-run-test post-run-test run-test-report run-test
    1119
    1120 ################################################################################
    1121 # Setup JCov
    1122 ################################################################################
    1123
    1124 ifeq ($(TEST_OPTS_JCOV), true)
    1125
    1126   jcov-do-start-grabber:
    1127 	$(call MakeDir, $(JCOV_OUTPUT_DIR))
    1128 	if $(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -status 1>/dev/null 2>&1 ; then \
    1129 	  $(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -stop -stoptimeout 3600 ; \
    1130 	fi
    1131 	$(JAVA) -Xmx4g -jar $(JCOV_HOME)/lib/jcov.jar Grabber -v -t \
    1132 	    $(JCOV_IMAGE_DIR)/template.xml -o $(JCOV_RESULT_FILE) \
    1133 	    1>$(JCOV_GRABBER_LOG) 2>&1 &
    1134
    1135   jcov-start-grabber: jcov-do-start-grabber
    1136 	$(call LogWarn, Starting JCov Grabber...)
    1137 	$(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -t 600 -wait
    1138
    1139   jcov-stop-grabber:
    1140 	$(call LogWarn, Stopping JCov Grabber...)
    1141 	$(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -stop -stoptimeout 3600
    1142
    1143   JCOV_REPORT_TITLE := JDK code coverage report<br/>
    1144   ifneq ($(JCOV_FILTERS), )
    1145     JCOV_REPORT_TITLE += Code filters: $(JCOV_FILTERS)<br>
    1146   endif
    1147   JCOV_REPORT_TITLE += Tests: $(TEST)
    1148
    1149   jcov-gen-report: jcov-stop-grabber
    1150 	$(call LogWarn, Generating JCov report ...)
    1151 	$(JAVA) -Xmx4g -jar $(JCOV_HOME)/lib/jcov.jar RepGen -sourcepath \
    1152 	    `$(ECHO) $(TOPDIR)/src/*/share/classes/ | $(TR) ' ' ':'` -fmt html \
    1153 	    $(JCOV_FILTERS) \
    1154 	    -mainReportTitle "$(JCOV_REPORT_TITLE)" \
    1155 	    -o $(JCOV_REPORT) $(JCOV_RESULT_FILE)
    1156
    1157   TARGETS += jcov-do-start-grabber jcov-start-grabber jcov-stop-grabber \
    1158       jcov-gen-report
    1159
    1160   ifneq ($(TEST_OPTS_JCOV_DIFF_CHANGESET), )
    1161
    1162     JCOV_SOURCE_DIFF := $(JCOV_OUTPUT_DIR)/source_diff
    1163     JCOV_DIFF_COVERAGE_REPORT := $(JCOV_OUTPUT_DIR)/diff_coverage_report
    1164
    1165     ifneq ($(and $(HG), $(wildcard $(TOPDIR)/.hg)), )
    1166       DIFF_COMMAND := $(HG) -R $(TOPDIR) diff -r $(TEST_OPTS_JCOV_DIFF_CHANGESET) > $(JCOV_SOURCE_DIFF)
    1167     else ifneq ($(and $(GIT), $(wildcard $(TOPDIR)/.git)), )
    1168       DIFF_COMMAND := $(GIT) -C $(TOPDIR) diff $(TEST_OPTS_JCOV_DIFF_CHANGESET) > $(JCOV_SOURCE_DIFF)
    1169     else
    1170       $(info Error: Must be either hg or git source tree for diff coverage.)
    1171       $(error Neither hg nor git source tree.)
    1172     endif
    1173
    1174     jcov-gen-diffcoverage: jcov-stop-grabber
    1175 	$(call LogWarn, Generating diff coverage with changeset $(TEST_OPTS_JCOV_DIFF_CHANGESET) ... )
    1176 	$(DIFF_COMMAND)
    1177 	$(JAVA) -Xmx4g -jar $(JCOV_HOME)/lib/jcov.jar \
    1178 	  DiffCoverage -replaceDiff "src/.*/classes/:" -all \
    1179 	    $(JCOV_RESULT_FILE) $(JCOV_SOURCE_DIFF) > \
    1180 	    $(JCOV_DIFF_COVERAGE_REPORT)
    1181
    1182     TARGETS += jcov-gen-diffcoverage
    1183
    1184   endif
    1185
    1186   # Hook this into the framework at appropriate places
    1187   pre-run-test: jcov-start-grabber
    1188
    1189   post-run-test: jcov-gen-report
    1190
    1191   ifneq ($(TEST_OPTS_JCOV_DIFF_CHANGESET), )
    1192
    1193     post-run-test: jcov-gen-diffcoverage
    1194
    1195   endif
    1196
    1197   jcov-stop-grabber: run-all-tests
    1198
    1199 endif
    1200
    1201 ################################################################################
    1202
    1203 all: run-test
    1204
    1205 .PHONY: default all $(TARGETS)