Build: Detect -fsanitize= in CFLAGS and incompatible build options.

Now configure will fail if -fsanitize= is found in CFLAGS
and sanitizer-incompatible ifunc or Landlock sandboxing
would be used. These are incompatible with one or more sanitizers.
It's simpler to reject all -fsanitize= uses instead of trying to
pass those that might not cause problems.

CMake-based build was updated similarly. It lets the configuration
finish (SEND_ERROR instead of FATAL_ERROR) so that both error
messages can be seen at once.

2 files changed, 62 insertions, 4 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 58cf62af..00071103 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -806,6 +806,14 @@ if(ALLOW_ATTR_IFUNC)
         HAVE_FUNC_ATTRIBUTE_IFUNC)
     cmake_pop_check_state()
     tuklib_add_definition_if(liblzma HAVE_FUNC_ATTRIBUTE_IFUNC)
+
+    if(HAVE_FUNC_ATTRIBUTE_IFUNC AND CMAKE_C_FLAGS MATCHES "-fsanitize=")
+        message(SEND_ERROR
+                "CMAKE_C_FLAGS or the environment variable CFLAGS "
+                "contains '-fsanitize=' which is incompatible "
+                "with ifunc. Use -DALLOW_ATTR_IFUNC=OFF "
+                "as an argument to 'cmake' when using '-fsanitize'.")
+    endif()
 endif()
 
 # cpuid.h
@@ -1293,9 +1301,30 @@ if(NOT MSVC OR MSVC_VERSION GREATER_EQUAL 1900)
     # Sandboxing: Landlock
     if(NOT SANDBOX_FOUND AND ENABLE_SANDBOX MATCHES "^ON$|^landlock$")
         check_include_file(linux/landlock.h HAVE_LINUX_LANDLOCK_H)
+
         if(HAVE_LINUX_LANDLOCK_H)
             target_compile_definitions(xz PRIVATE HAVE_LINUX_LANDLOCK_H)
             set(SANDBOX_FOUND ON)
+
+            # Of our three sandbox methods, only Landlock is incompatible
+            # with -fsanitize. FreeBSD 13.2 with Capsicum was tested with
+            # -fsanitize=address,undefined and had no issues. OpenBSD (as
+            # of version 7.4) has minimal support for process instrumentation.
+            # OpenBSD does not distribute the additional libraries needed
+            # (libasan, libubsan, etc.) with GCC or Clang needed for runtime
+            # sanitization support and instead only support
+            # -fsanitize-minimal-runtime for minimal undefined behavior
+            # sanitization. This minimal support is compatible with our use
+            # of the Pledge sandbox. So only Landlock will result in a
+            # build that cannot compress or decompress a single file to
+            # standard out.
+            if(CMAKE_C_FLAGS MATCHES "-fsanitize=")
+                message(SEND_ERROR
+                        "CMAKE_C_FLAGS or the environment variable CFLAGS "
+                        "contains '-fsanitize=' which is incompatible "
+                        "with Landlock sandboxing. Use -DENABLE_SANDBOX=OFF "
+                        "as an argument to 'cmake' when using '-fsanitize'.")
+            endif()
         endif()
     endif()
 
diff --git a/configure.ac b/configure.ac
index 00a9e3c0..553a1b87 100644
--- a/configure.ac
+++ b/configure.ac
@@ -523,11 +523,15 @@ AC_ARG_ENABLE([sandbox], [AS_HELP_STRING([--enable-sandbox=METHOD],
 		The default is 'auto' which enables sandboxing if
 		a supported sandboxing method is found.])],
 	[], [enable_sandbox=auto])
-case $enable_sandbox in
-	auto)
+case $enable_xz-$enable_sandbox in
+	no-*)
+		enable_sandbox=no
+		AC_MSG_RESULT([no, --disable-xz was used])
+		;;
+	*-auto)
 		AC_MSG_RESULT([maybe (autodetect)])
 		;;
-	no | capsicum | pledge | landlock)
+	*-no | *-capsicum | *-pledge | *-landlock)
 		AC_MSG_RESULT([$enable_sandbox])
 		;;
 	*)
@@ -890,6 +894,14 @@ if test "x$enable_ifunc" = xyes ; then
 			[Define to 1 if __attribute__((__ifunc__()))
 			is supported for functions.])
 		AC_MSG_RESULT([yes])
+
+		# ifunc explicitly does not work with -fsanitize=address.
+		# If configured, it will result in a liblzma build that
+		# will fail when liblzma is loaded at runtime (when the
+		# ifunc resolver executes).
+		AS_CASE([$CFLAGS], [*-fsanitize=*], [AC_MSG_ERROR([
+    CFLAGS contains '-fsanitize=' which is incompatible with ifunc.
+    Use --disable-ifunc when using '-fsanitize'.])])
 	], [
 		AC_MSG_RESULT([no])
 	])
@@ -1049,6 +1061,17 @@ __m128i my_clmul(__m128i a)
 AM_CONDITIONAL([COND_CRC_CLMUL], [test "x$enable_clmul_crc" = xyes])
 
 # Check for sandbox support. If one is found, set enable_sandbox=found.
+#
+# About -fsanitize: Of our three sandbox methods, only Landlock is
+# incompatible with -fsanitize. FreeBSD 13.2 with Capsicum was tested with
+# -fsanitize=address,undefined and had no issues. OpenBSD (as of version
+# 7.4) has minimal support for process instrumentation. OpenBSD does not
+# distribute the additional libraries needed (libasan, libubsan, etc.) with
+# GCC or Clang needed for runtime sanitization support and instead only
+# support -fsanitize-minimal-runtime for minimal undefined behavior
+# sanitization. This minimal support is compatible with our use of the
+# Pledge sandbox. So only Landlock will result in a build that cannot
+# compress or decompress a single file to standard out.
 AS_CASE([$enable_sandbox],
 	[auto | capsicum], [
 		AC_CHECK_FUNCS([cap_rights_limit], [enable_sandbox=found])
@@ -1061,7 +1084,13 @@ AS_CASE([$enable_sandbox],
 )
 AS_CASE([$enable_sandbox],
 	[auto | landlock], [
-		AC_CHECK_HEADERS([linux/landlock.h], [enable_sandbox=found])
+		AC_CHECK_HEADERS([linux/landlock.h], [
+			enable_sandbox=found
+
+			AS_CASE([$CFLAGS], [*-fsanitize=*], [AC_MSG_ERROR([
+    CFLAGS contains '-fsanitize=' which is incompatible with the Landlock
+    sandboxing. Use --disable-sandbox when using '-fsanitize'.])])
+		])
 	]
 )