Has only seen with linux-aarch64, both release and debug builds, intermittent. Has only been seen with -XX:+UseZGC and -Xcomp so far.

The log snippet:

#
# There is insufficient memory for the Java Runtime Environment to continue.
# Native memory allocation (malloc) failed to allocate 8 bytes for AllocateHeap
# An error report file with more information is saved as:
# /opt/mach5/mesos/work_dir/slaves/91e16c40-06d4-468a-9fc3-7198a5bb7d5a-S35216/frameworks/1735e8a2-a1db-478c-8104-60c8b0af87dd-0196/executors/a034b3ce-df50-429e-bcb2-501e0d8b998b/runs/4cd65b41-552f-4089-ae0c-3d08520f2878/testoutput/test-support/jtreg_open_test_hotspot_jtreg_vmTestbase_nsk_jvmti/scratch/0/hs_err_pid30753.log
#
# Can't open file to dump replay data. Error: Not enough space

[2022-12-10T05:18:03.685174978Z] Waiting for completion for process 30752
[2022-12-10T05:18:03.688706648Z] Waiting for completion finished for process 30752
[2022-12-10T05:18:03.689959765Z] Waiting for completion for process 30752
[2022-12-10T05:18:03.689970645Z] Waiting for completion finished for process 30752
[2022-12-10T05:18:03.689988205Z] Waiting for completion for process 30752
[2022-12-10T05:18:03.689992165Z] Waiting for completion finished for process 30752
[2022-12-10T05:18:03.709667792Z] Waiting for completion for process 30752
[2022-12-10T05:18:03.709691272Z] Waiting for completion finished for process 30752
----------System.err:(22/1309)----------
 stdout: [#
# There is insufficient memory for the Java Runtime Environment to continue.
# Native memory allocation (malloc) failed to allocate 8 bytes for AllocateHeap
# An error report file with more information is saved as:
# /opt/mach5/mesos/work_dir/slaves/91e16c40-06d4-468a-9fc3-7198a5bb7d5a-S35216/frameworks/1735e8a2-a1db-478c-8104-60c8b0af87dd-0196/executors/a034b3ce-df50-429e-bcb2-501e0d8b998b/runs/4cd65b41-552f-4089-ae0c-3d08520f2878/testoutput/test-support/jtreg_open_test_hotspot_jtreg_vmTestbase_nsk_jvmti/scratch/0/hs_err_pid30753.log
#
# Can't open file to dump replay data. Error: Not enough space
];
 stderr: []
 exitValue = 1

java.lang.RuntimeException: Expected to get exit value of [95], exit value is: [1]
at jdk.test.lib.process.OutputAnalyzer.shouldHaveExitValue(OutputAnalyzer.java:490)
at nsk.jvmti.Allocate.alloc001.alloc001.main(alloc001.java:105)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base/java.lang.reflect.Method.invoke(Method.java:578)
at com.sun.javatest.regtest.agent.MainActionHelper$AgentVMRunnable.run(MainActionHelper.java:312)
at java.base/java.lang.Thread.run(Thread.java:1623)