StackOverflowError is thrown when executing Runtime.exec() by JNI after glog is initialized.
Exception in thread "process reaper" Done spawn.
java.lang.StackOverflowError

This issue does not occur in JDK 8 but both JDK 17 and 21 have this issue.



STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
1. Build glog and gflags to static libs (or shared libs).
2. Build ForkTest.java by "javac ForkTest.java".
3. Build ForkTest.cpp by
clang++ ForkTest.cpp -I/<PATH>/jdk-17.0.5/include -I/<PATH>/jdk-17.0.5/include/linux -I/<PATH>/include/ -lglog -lgflags -L/<PATH>/<GLOG_DIR>/build/ -L/<PATH>/<GFLAGS_DIR>/build/lib -ldl -lpthread -o forktest
4. export JAVA_HOME=''
5. forktest java
Run multiple times, the error will occur sometime.


EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
No error.
ACTUAL -
Exception in thread "process reaper" Done spawn.
java.lang.StackOverflowError
at java.base/jdk.internal.misc.Unsafe.compareAndSetLong(Native Method)
at java.base/java.util.concurrent.ConcurrentHashMap.addCount(ConcurrentHashMap.java:2327)
at java.base/java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1075)
at java.base/java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1541)
at java.base/java.lang.invoke.MethodType$ConcurrentWeakInternSet.add(MethodType.java:1398)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:357)
at java.base/java.lang.invoke.MethodTypeForm.<init>(MethodTypeForm.java:197)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:223)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodTypeForm.canonicalize(MethodTypeForm.java:253)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:220)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodType.insertParameterTypes(MethodType.java:448)
at java.base/java.lang.invoke.VarHandle$AccessDescriptor.<init>(VarHandle.java:1982)
at java.base/java.lang.invoke.MethodHandleNatives.varHandleOperationLinkerMethod(MethodHandleNatives.java:541)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethodImpl(MethodHandleNatives.java:485)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethod(MethodHandleNatives.java:473)
at java.base/java.util.concurrent.CompletableFuture.completeValue(CompletableFuture.java:306)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2146)
at java.base/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:171)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)



---------- BEGIN SOURCE ----------
$ cat ForkTest.cpp
#include <jni.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <memory>
#include <mutex>
#include <filesystem>
#include <dlfcn.h>
#include <glog/logging.h>

class LibJVMLoader {
public:
    LibJVMLoader(const LibJVMLoader&) = delete;
    LibJVMLoader& operator=(const LibJVMLoader&) = delete;

    static LibJVMLoader& instance();
    int load();

    using JNI_GetCreatedJavaVMsPointer = std::add_pointer_t<decltype(::JNI_GetCreatedJavaVMs)>;
    static JNI_GetCreatedJavaVMsPointer JNI_GetCreatedJavaVMs;

    using JNI_CreateJavaVMPointer = std::add_pointer_t<decltype(::JNI_CreateJavaVM)>;
    static JNI_CreateJavaVMPointer JNI_CreateJavaVM;

private:
    explicit LibJVMLoader(std::string_view library)
            : _library(library), _handle(nullptr, nullptr) {}

    const std::string _library;
    std::unique_ptr<void, void (*)(void*)> _handle;
};


_JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_GetCreatedJavaVMs(JavaVM** vm_buf, jsize bufLen,
                                                          jsize* numVMs) {
    return LibJVMLoader::JNI_GetCreatedJavaVMs(vm_buf, bufLen, numVMs);
}

_JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_CreateJavaVM(JavaVM** pvm, void** penv, void* args) {
    return LibJVMLoader::JNI_CreateJavaVM(pvm, penv, args);
}

namespace {

#ifndef __APPLE__
#define LIBJVM_SO "libjvm.so"
#else
#define LIBJVM_SO "libjvm.dylib"
#endif

template <typename T>
int resolve_symbol(T& pointer, void* handle, const char* symbol) {
    pointer = reinterpret_cast<T>(dlsym(handle, symbol));
    return (pointer != nullptr)
                   ? 0
                   : -1;
}

} // namespace


LibJVMLoader::JNI_GetCreatedJavaVMsPointer LibJVMLoader::JNI_GetCreatedJavaVMs = nullptr;
LibJVMLoader::JNI_CreateJavaVMPointer LibJVMLoader::JNI_CreateJavaVM = nullptr;

LibJVMLoader& LibJVMLoader::instance() {
    static std::once_flag find_library;
    static std::string library;
    std::call_once(find_library, []() {
        const auto* java_home = getenv("JAVA_HOME");
        if (!java_home) {
            return;
        }
        std::string path(java_home);
        for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) {
            if (entry.path().filename() == LIBJVM_SO) {
                library = entry.path().string();
                break;
            }
        }
     fprintf(stderr, "path: %s\n", path.c_str());
    });

    static LibJVMLoader loader(library);
    return loader;
}

int LibJVMLoader::load() {
    if (_library.empty()) {
     fprintf(stderr, "_library is empty\n");
        return -1;
    }

    static std::once_flag resolve_symbols;
    static int status;
    std::call_once(resolve_symbols, [this]() {
        _handle = std::unique_ptr<void, void (*)(void*)>(dlopen(_library.c_str(), RTLD_LAZY),
                                                         [](void* handle) { dlclose(handle); });
        if (!_handle) {
            status = -1;
            return;
        }

        if (-1 == resolve_symbol(JNI_GetCreatedJavaVMs, _handle.get(), "JNI_GetCreatedJavaVMs")) {
            status = -1;
            return;
        }
        if (-1 == resolve_symbol(JNI_CreateJavaVM, _handle.get(), "JNI_CreateJavaVM")) {
            status = -1;
            return;
        }
    });
    return status;
}

// Create the Java virtual machine, and start a test object.
static JNIEnv* createJVM(void) {
    if (LibJVMLoader::instance().load() < 0) {
     return nullptr;
    }
    JavaVM* jvm;
    JNIEnv* env;

    JavaVMOption options[20];
    int numOptions = 0;
    options[numOptions++].optionString = "-Djava.class.path=.";
    options[numOptions++].optionString = "-Xss1m";

    JavaVMInitArgs args;
    args.version = JNI_VERSION_1_2;
    args.options = options;
    args.nOptions = (jint)numOptions;
    args.ignoreUnrecognized = 0;

    if (JNI_CreateJavaVM(&jvm, (void **)&env, &args)) {
     fprintf(stderr, "Can't create JVM.\n");
    }
    fprintf(stderr, "JVM created.\n");
    return env;
}

void testJavaFork(JNIEnv *env) {
    jclass clazz = env->FindClass("ForkTest");
    if (env->ExceptionOccurred() || (clazz == 0)) {
     fprintf(stderr, "Can't find Java class.\n");
        exit(1);
    }
    jmethodID init = env->GetMethodID(clazz, "<init>", "()V");
    if (env->ExceptionOccurred() || (init == 0)) {
     fprintf(stderr, "Can't find Java method.\n");
        exit(1);
    }
    jobject obj = env->NewObject(clazz, init);
    if (env->ExceptionOccurred() || (obj == 0)) {
     fprintf(stderr, "Java object created.\n");
        exit(1);
    }
    fprintf(stderr, "Java object created.\n");
}

void testCFork(void) {
    pid_t pid;
    if ((pid = fork()) == 0) {
        // Child process
        execl("/bin/ls", "ls", (char *)0);
     fprintf(stderr, "Error in C exec.\n");
        exit(1);
    }
    // Parent process
    fprintf(stderr, "Child PID = %d\n", pid);
}

int main(int argc, char **argv) {
    google::LogToStderr();
    google::InitGoogleLogging("fork_test");
    LOG(INFO) << "hello world";
    JNIEnv* env = createJVM();
    if (!env) {
     exit(1);
    }

    if (argc > 1) {
        testJavaFork(env);
    } else {
        testCFork();
    }

    LOG(INFO) << "Waiting...";
    sleep(3);
    LOG(INFO) << "Running Successfully.";
    google::ShutdownGoogleLogging();
    return 0;
}

$ cat ForkTest.java
import java.io.*;

public class ForkTest {
    public static void main(String[] args) throws Exception {
new ForkTest();
    }

    public ForkTest() {
exec();
    }

    /**
     * Spawn a process to run a ticker shell script.
     */
    public void exec() {
        try {
            System.out.println("Java Fork Test....");
            Process proc = Runtime.getRuntime().exec("ls");
            proc.waitFor();
            System.out.println("Java ForkTest exit status = " + proc.exitValue());
        } catch (Exception e) {
            System.err.println("Java ForkTest failed " + e.getMessage());
            return;
        }
    }
}

---------- END SOURCE ----------

CUSTOMER SUBMITTED WORKAROUND :
The root cause is that the glog allocated and initialized some vars described by following source code in <GLOG_DIR>/src/logging.cc.
The workaround is reduce memory usage by decrease LogMessage::kMaxLogMessageLen from 30000 to 1024.

const size_t LogMessage::kMaxLogMessageLen = 30000;
struct LogMessage::LogMessageData {
  LogMessageData();

  int preserved_errno_; // preserved errno
  // Buffer space; contains complete message text.
  char message_text_[LogMessage::kMaxLogMessageLen+1];
  LogStream stream_;
  char severity_; // What level is this LogMessage logged at?
  int line_; // line number where logging call is.
  void (LogMessage::*send_method_)(); // Call this in destructor to send
  union { // At most one of these is used: union to keep the size low.
    LogSink* sink_; // NULL or sink to send message to
    std::vector<std::string>* outvec_; // NULL or vector to push message onto
    std::string* message_; // NULL or string to write message into
  };
  size_t num_prefix_chars_; // # of chars of prefix in this message
  size_t num_chars_to_log_; // # of chars of msg to send to log
  size_t num_chars_to_syslog_; // # of chars of msg to send to syslog
  const char* basename_; // basename of file that called LOG
  const char* fullname_; // fullname of file that called LOG
  bool has_been_flushed_; // false => data has not been flushed
  bool first_fatal_; // true => this was first fatal msg

 private:
  LogMessageData(const LogMessageData&);
  void operator=(const LogMessageData&);
};

static LogMessage::LogMessageData fatal_msg_data_exclusive;
static LogMessage::LogMessageData fatal_msg_data_shared;