java-17-openjdk/rh1929465-improve_system_FIPS_detection.patch

diff --git openjdk/make/autoconf/lib-sysconf.m4 openjdk/make/autoconf/lib-sysconf.m4
new file mode 100644
index 00000000000..b2b1c1787da
--- /dev/null
+++ openjdk/make/autoconf/lib-sysconf.m4
@@ -0,0 +1,84 @@
+#
+# Copyright (c) 2021, Red Hat, Inc.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# Setup system configuration libraries
+################################################################################
+AC_DEFUN_ONCE([LIB_SETUP_SYSCONF_LIBS],
+[
+  ###############################################################################
+  #
+  # Check for the NSS library
+  #
+
+  AC_MSG_CHECKING([whether to use the system NSS library with the System Configurator (libsysconf)])
+
+  # default is not available
+  DEFAULT_SYSCONF_NSS=no
+
+  AC_ARG_ENABLE([sysconf-nss], [AS_HELP_STRING([--enable-sysconf-nss],
+     [build the System Configurator (libsysconf) using the system NSS library if available @<:@disabled@:>@])],
+  [
+    case "${enableval}" in
+      yes)
+        sysconf_nss=yes
+        ;;
+      *)
+        sysconf_nss=no
+        ;;
+    esac
+  ],
+  [
+    sysconf_nss=${DEFAULT_SYSCONF_NSS}
+  ])
+  AC_MSG_RESULT([$sysconf_nss])
+
+  USE_SYSCONF_NSS=false
+  if test "x${sysconf_nss}" = "xyes"; then
+      PKG_CHECK_MODULES(NSS, nss >= 3.53, [NSS_FOUND=yes], [NSS_FOUND=no])
+      if test "x${NSS_FOUND}" = "xyes"; then
+         AC_MSG_CHECKING([for system FIPS support in NSS])
+         saved_libs="${LIBS}"
+         saved_cflags="${CFLAGS}"
+         CFLAGS="${CFLAGS} ${NSS_CFLAGS}"
+         LIBS="${LIBS} ${NSS_LIBS}"
+         AC_LANG_PUSH([C])
+         AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <nss3/pk11pub.h>]],
+                                         [[SECMOD_GetSystemFIPSEnabled()]])],
+                        [AC_MSG_RESULT([yes])],
+                        [AC_MSG_RESULT([no])
+                        AC_MSG_ERROR([System NSS FIPS detection unavailable])])
+         AC_LANG_POP([C])
+         CFLAGS="${saved_cflags}"
+         LIBS="${saved_libs}"
+         USE_SYSCONF_NSS=true
+      else
+         dnl NSS 3.53 is the one that introduces the SECMOD_GetSystemFIPSEnabled API
+         dnl in nss3/pk11pub.h.
+         AC_MSG_ERROR([--enable-sysconf-nss specified, but NSS 3.53 or above not found.])
+      fi
+  fi
+  AC_SUBST(USE_SYSCONF_NSS)
+])
diff --git openjdk/make/autoconf/libraries.m4 openjdk/make/autoconf/libraries.m4
index a65d91ee974..a8f054c1397 100644
--- openjdk/make/autoconf/libraries.m4
+++ openjdk/make/autoconf/libraries.m4
@@ -33,6 +33,7 @@ m4_include([lib-std.m4])
 m4_include([lib-x11.m4])
 m4_include([lib-fontconfig.m4])
 m4_include([lib-tests.m4])
+m4_include([lib-sysconf.m4])
 
 ################################################################################
 # Determine which libraries are needed for this configuration
@@ -104,6 +105,7 @@ AC_DEFUN_ONCE([LIB_SETUP_LIBRARIES],
   LIB_SETUP_BUNDLED_LIBS
   LIB_SETUP_MISC_LIBS
   LIB_TESTS_SETUP_GTEST
+  LIB_SETUP_SYSCONF_LIBS
 
   BASIC_JDKLIB_LIBS=""
   if test "x$TOOLCHAIN_TYPE" != xmicrosoft; then
diff --git openjdk/make/autoconf/spec.gmk.in openjdk/make/autoconf/spec.gmk.in
index 29445c8c24f..9b1b512a34a 100644
--- openjdk/make/autoconf/spec.gmk.in
+++ openjdk/make/autoconf/spec.gmk.in
@@ -834,6 +834,10 @@ INSTALL_SYSCONFDIR=@sysconfdir@
 # Libraries
 #
 
+USE_SYSCONF_NSS:=@USE_SYSCONF_NSS@
+NSS_LIBS:=@NSS_LIBS@
+NSS_CFLAGS:=@NSS_CFLAGS@
+
 USE_EXTERNAL_LCMS:=@USE_EXTERNAL_LCMS@
 LCMS_CFLAGS:=@LCMS_CFLAGS@
 LCMS_LIBS:=@LCMS_LIBS@
diff --git openjdk/make/modules/java.base/Lib.gmk openjdk/make/modules/java.base/Lib.gmk
index 5658ff342e5..cb7a56852f7 100644
--- openjdk/make/modules/java.base/Lib.gmk
+++ openjdk/make/modules/java.base/Lib.gmk
@@ -167,6 +167,31 @@ ifeq ($(call isTargetOsType, unix), true)
   endif
 endif
 
+################################################################################
+# Create the systemconf library
+
+LIBSYSTEMCONF_CFLAGS :=
+LIBSYSTEMCONF_CXXFLAGS :=
+
+ifeq ($(USE_SYSCONF_NSS), true)
+  LIBSYSTEMCONF_CFLAGS += $(NSS_CFLAGS) -DSYSCONF_NSS
+  LIBSYSTEMCONF_CXXFLAGS += $(NSS_CFLAGS) -DSYSCONF_NSS
+endif
+
+ifeq ($(OPENJDK_BUILD_OS), linux)
+  $(eval $(call SetupJdkLibrary, BUILD_LIBSYSTEMCONF, \
+      NAME := systemconf, \
+      OPTIMIZATION := LOW, \
+      CFLAGS := $(CFLAGS_JDKLIB) $(LIBSYSTEMCONF_CFLAGS), \
+      CXXFLAGS := $(CXXFLAGS_JDKLIB) $(LIBSYSTEMCONF_CXXFLAGS), \
+      LDFLAGS := $(LDFLAGS_JDKLIB) \
+          $(call SET_SHARED_LIBRARY_ORIGIN), \
+      LIBS_unix := $(LIBDL) $(NSS_LIBS), \
+  ))
+
+  TARGETS += $(BUILD_LIBSYSTEMCONF)
+endif
+
 ################################################################################
 # Create the symbols file for static builds.
 
diff --git openjdk/src/java.base/linux/native/libsystemconf/systemconf.c openjdk/src/java.base/linux/native/libsystemconf/systemconf.c
new file mode 100644
index 00000000000..6f4656bfcb6
--- /dev/null
+++ openjdk/src/java.base/linux/native/libsystemconf/systemconf.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2021, Red Hat, Inc.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <dlfcn.h>
+#include <jni.h>
+#include <jni_util.h>
+#include <stdio.h>
+
+#ifdef SYSCONF_NSS
+#include <nss3/pk11pub.h>
+#endif //SYSCONF_NSS
+
+#include "java_security_SystemConfigurator.h"
+
+#define FIPS_ENABLED_PATH "/proc/sys/crypto/fips_enabled"
+#define MSG_MAX_SIZE 96
+
+static jmethodID debugPrintlnMethodID = NULL;
+static jobject debugObj = NULL;
+
+static void throwIOException(JNIEnv *env, const char *msg);
+static void dbgPrint(JNIEnv *env, const char* msg);
+
+/*
+ * Class:     java_security_SystemConfigurator
+ * Method:    JNI_OnLoad
+ */
+JNIEXPORT jint JNICALL DEF_JNI_OnLoad(JavaVM *vm, void *reserved)
+{
+    JNIEnv *env;
+    jclass sysConfCls, debugCls;
+    jfieldID sdebugFld;
+
+    if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_2) != JNI_OK) {
+        return JNI_EVERSION; /* JNI version not supported */
+    }
+
+    sysConfCls = (*env)->FindClass(env,"java/security/SystemConfigurator");
+    if (sysConfCls == NULL) {
+        printf("libsystemconf: SystemConfigurator class not found\n");
+        return JNI_ERR;
+    }
+    sdebugFld = (*env)->GetStaticFieldID(env, sysConfCls,
+            "sdebug", "Lsun/security/util/Debug;");
+    if (sdebugFld == NULL) {
+        printf("libsystemconf: SystemConfigurator::sdebug field not found\n");
+        return JNI_ERR;
+    }
+    debugObj = (*env)->GetStaticObjectField(env, sysConfCls, sdebugFld);
+    if (debugObj != NULL) {
+        debugCls = (*env)->FindClass(env,"sun/security/util/Debug");
+        if (debugCls == NULL) {
+            printf("libsystemconf: Debug class not found\n");
+            return JNI_ERR;
+        }
+        debugPrintlnMethodID = (*env)->GetMethodID(env, debugCls,
+                "println", "(Ljava/lang/String;)V");
+        if (debugPrintlnMethodID == NULL) {
+            printf("libsystemconf: Debug::println(String) method not found\n");
+            return JNI_ERR;
+        }
+        debugObj = (*env)->NewGlobalRef(env, debugObj);
+    }
+
+    return (*env)->GetVersion(env);
+}
+
+/*
+ * Class:     java_security_SystemConfigurator
+ * Method:    JNI_OnUnload
+ */
+JNIEXPORT void JNICALL DEF_JNI_OnUnload(JavaVM *vm, void *reserved)
+{
+    JNIEnv *env;
+
+    if (debugObj != NULL) {
+        if ((*vm)->GetEnv(vm, (void**) &env, JNI_VERSION_1_2) != JNI_OK) {
+            return; /* Should not happen */
+        }
+        (*env)->DeleteGlobalRef(env, debugObj);
+    }
+}
+
+JNIEXPORT jboolean JNICALL Java_java_security_SystemConfigurator_getSystemFIPSEnabled
+  (JNIEnv *env, jclass cls)
+{
+    int fips_enabled;
+    char msg[MSG_MAX_SIZE];
+    int msg_bytes;
+
+#ifdef SYSCONF_NSS
+
+    dbgPrint(env, "getSystemFIPSEnabled: calling SECMOD_GetSystemFIPSEnabled");
+    fips_enabled = SECMOD_GetSystemFIPSEnabled();
+    msg_bytes = snprintf(msg, MSG_MAX_SIZE, "getSystemFIPSEnabled:" \
+            " SECMOD_GetSystemFIPSEnabled returned 0x%x", fips_enabled);
+    if (msg_bytes > 0 && msg_bytes < MSG_MAX_SIZE) {
+        dbgPrint(env, msg);
+    } else {
+        dbgPrint(env, "getSystemFIPSEnabled: cannot render" \
+                " SECMOD_GetSystemFIPSEnabled return value");
+    }
+    return (fips_enabled == 1 ? JNI_TRUE : JNI_FALSE);
+
+#else // SYSCONF_NSS
+
+    FILE *fe;
+
+    dbgPrint(env, "getSystemFIPSEnabled: reading " FIPS_ENABLED_PATH);
+    if ((fe = fopen(FIPS_ENABLED_PATH, "r")) == NULL) {
+        throwIOException(env, "Cannot open " FIPS_ENABLED_PATH);
+    }
+    fips_enabled = fgetc(fe);
+    fclose(fe);
+    if (fips_enabled == EOF) {
+        throwIOException(env, "Cannot read " FIPS_ENABLED_PATH);
+    }
+    msg_bytes = snprintf(msg, MSG_MAX_SIZE, "getSystemFIPSEnabled:" \
+            " read character is '%c'", fips_enabled);
+    if (msg_bytes > 0 && msg_bytes < MSG_MAX_SIZE) {
+        dbgPrint(env, msg);
+    } else {
+        dbgPrint(env, "getSystemFIPSEnabled: cannot render" \
+                " read character");
+    }
+    return (fips_enabled == '1' ? JNI_TRUE : JNI_FALSE);
+
+#endif // SYSCONF_NSS
+}
+
+static void throwIOException(JNIEnv *env, const char *msg)
+{
+    jclass cls = (*env)->FindClass(env, "java/io/IOException");
+    if (cls != 0)
+        (*env)->ThrowNew(env, cls, msg);
+}
+
+static void dbgPrint(JNIEnv *env, const char* msg)
+{
+    jstring jMsg;
+    if (debugObj != NULL) {
+        jMsg = (*env)->NewStringUTF(env, msg);
+        CHECK_NULL(jMsg);
+        (*env)->CallVoidMethod(env, debugObj, debugPrintlnMethodID, jMsg);
+    }
+}
diff --git openjdk/src/java.base/share/classes/java/security/SystemConfigurator.java openjdk/src/java.base/share/classes/java/security/SystemConfigurator.java
index 10b54aa4ce4..6aa1419dfd0 100644
--- openjdk/src/java.base/share/classes/java/security/SystemConfigurator.java
+++ openjdk/src/java.base/share/classes/java/security/SystemConfigurator.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Red Hat, Inc.
+ * Copyright (c) 2019, 2021, Red Hat, Inc.
  *
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -30,13 +30,9 @@ import java.io.BufferedInputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
 
-import java.nio.file.Files;
-import java.nio.file.Path;
-
 import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.Properties;
-import java.util.regex.Pattern;
 
 import sun.security.util.Debug;
 
@@ -58,11 +54,23 @@ final class SystemConfigurator {
     private static final String CRYPTO_POLICIES_JAVA_CONFIG =
             CRYPTO_POLICIES_BASE_DIR + "/back-ends/java.config";
 
-    private static final String CRYPTO_POLICIES_CONFIG =
-            CRYPTO_POLICIES_BASE_DIR + "/config";
-
     private static boolean systemFipsEnabled = false;
 
+    private static final String SYSTEMCONF_NATIVE_LIB = "systemconf";
+
+    private static native boolean getSystemFIPSEnabled()
+            throws IOException;
+
+    static {
+        @SuppressWarnings("removal")
+        var dummy = AccessController.doPrivileged(new PrivilegedAction<Void>() {
+            public Void run() {
+                System.loadLibrary(SYSTEMCONF_NATIVE_LIB);
+                return null;
+            }
+        });
+    }
+
     /*
      * Invoked when java.security.Security class is initialized, if
      * java.security.disableSystemPropertiesFile property is not set and
@@ -170,16 +178,34 @@ final class SystemConfigurator {
     }
 
     /*
-     * FIPS is enabled only if crypto-policies are set to "FIPS"
-     * and the com.redhat.fips property is true.
+     * OpenJDK FIPS mode will be enabled only if the com.redhat.fips
+     * system property is true (default) and the system is in FIPS mode.
+     *
+     * There are 2 possible ways in which OpenJDK detects that the system
+     * is in FIPS mode: 1) if the NSS SECMOD_GetSystemFIPSEnabled API is
+     * available at OpenJDK's built-time, it is called; 2) otherwise, the
+     * /proc/sys/crypto/fips_enabled file is read.
      */
     private static boolean enableFips() throws Exception {
         boolean shouldEnable = Boolean.valueOf(System.getProperty("com.redhat.fips", "true"));
         if (shouldEnable) {
-            String cryptoPoliciesConfig = new String(Files.readAllBytes(Path.of(CRYPTO_POLICIES_CONFIG)));
-            if (sdebug != null) { sdebug.println("Crypto config:\n" + cryptoPoliciesConfig); }
-            Pattern pattern = Pattern.compile("^FIPS$", Pattern.MULTILINE);
-            return pattern.matcher(cryptoPoliciesConfig).find();
+            if (sdebug != null) {
+                sdebug.println("Calling getSystemFIPSEnabled (libsystemconf)...");
+            }
+            try {
+                shouldEnable = getSystemFIPSEnabled();
+                if (sdebug != null) {
+                    sdebug.println("Call to getSystemFIPSEnabled (libsystemconf) returned: "
+                            + shouldEnable);
+                }
+                return shouldEnable;
+            } catch (IOException e) {
+                if (sdebug != null) {
+                    sdebug.println("Call to getSystemFIPSEnabled (libsystemconf) failed:");
+                    sdebug.println(e.getMessage());
+                }
+                throw e;
+            }
         } else {
             return false;
         }