ffmpeg/ffmpeg-dlopen-openh264.patch

From b8886826f44b4562aa2fbae3715ca6aa9e1b15e1 Mon Sep 17 00:00:00 2001
From: Neal Gompa <ngompa@fedoraproject.org>
Date: Wed, 12 Oct 2022 10:26:35 -0400
Subject: [PATCH] avcodec/openh264: Add the ability to dlopen() OpenH264

We can't directly depend on OpenH264, but we can weakly link to it
and gracefully expose the capability.

Co-authored-by: Andreas Schneider <asn@cryptomilk.org>
Co-authored-by: Neal Gompa <ngompa@fedoraproject.org>

Signed-off-by: Andreas Schneider <asn@cryptomilk.org>
Signed-off-by: Neal Gompa <ngompa@fedoraproject.org>
---
 configure                       |   3 +
 libavcodec/Makefile             |   1 +
 libavcodec/libopenh264.c        |  18 +++-
 libavcodec/libopenh264_dlopen.c | 147 ++++++++++++++++++++++++++++++++
 libavcodec/libopenh264_dlopen.h |  58 +++++++++++++
 libavcodec/libopenh264dec.c     |  10 +++
 libavcodec/libopenh264enc.c     |  10 +++
 7 files changed, 245 insertions(+), 2 deletions(-)
 create mode 100644 libavcodec/libopenh264_dlopen.c
 create mode 100644 libavcodec/libopenh264_dlopen.h

diff --git a/configure b/configure
index 7a62f0c248..2369ea0240 100755
--- a/configure
+++ b/configure
@@ -250,6 +250,7 @@ External library support:
   --enable-libopencore-amrwb enable AMR-WB decoding via libopencore-amrwb [no]
   --enable-libopencv       enable video filtering via libopencv [no]
   --enable-libopenh264     enable H.264 encoding via OpenH264 [no]
+  --enable-libopenh264-dlopen  enable H.264 encoding via dlopen()'ed OpenH264 [no]
   --enable-libopenjpeg     enable JPEG 2000 de/encoding via OpenJPEG [no]
   --enable-libopenmpt      enable decoding tracked files via libopenmpt [no]
   --enable-libopenvino     enable OpenVINO as a DNN module backend
@@ -1839,6 +1840,7 @@ EXTERNAL_LIBRARY_LIST="
     libmysofa
     libopencv
     libopenh264
+    libopenh264_dlopen
     libopenjpeg
     libopenmpt
     libopenvino
@@ -6582,6 +6584,7 @@ enabled libopencv         && { check_headers opencv2/core/core_c.h &&
                                  require libopencv opencv2/core/core_c.h cvCreateImageHeader -lopencv_core -lopencv_imgproc; } ||
                                require_pkg_config libopencv opencv opencv/cxcore.h cvCreateImageHeader; }
 enabled libopenh264       && require_pkg_config libopenh264 openh264 wels/codec_api.h WelsGetCodecVersion
+enabled libopenh264_dlopen && enable libopenh264 && add_cppflags "-I$(dirname `readlink -f $0`)/ffdlopenhdrs/include -DCONFIG_LIBOPENH264_DLOPEN=1"
 enabled libopenjpeg       && { check_pkg_config libopenjpeg "libopenjp2 >= 2.1.0" openjpeg.h opj_version ||
                                { require_pkg_config libopenjpeg "libopenjp2 >= 2.1.0" openjpeg.h opj_version -DOPJ_STATIC && add_cppflags -DOPJ_STATIC; } }
 enabled libopenmpt        && require_pkg_config libopenmpt "libopenmpt >= 0.2.6557" libopenmpt/libopenmpt.h openmpt_module_create -lstdc++ && append libopenmpt_extralibs "-lstdc++"
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 3adf1536d8..9dc8389ce2 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1058,6 +1058,7 @@ OBJS-$(CONFIG_LIBMP3LAME_ENCODER)         += libmp3lame.o
 OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER)  += libopencore-amr.o
 OBJS-$(CONFIG_LIBOPENCORE_AMRNB_ENCODER)  += libopencore-amr.o
 OBJS-$(CONFIG_LIBOPENCORE_AMRWB_DECODER)  += libopencore-amr.o
+OBJS-$(CONFIG_LIBOPENH264_DLOPEN)         += libopenh264_dlopen.o
 OBJS-$(CONFIG_LIBOPENH264_DECODER)        += libopenh264dec.o libopenh264.o
 OBJS-$(CONFIG_LIBOPENH264_ENCODER)        += libopenh264enc.o libopenh264.o
 OBJS-$(CONFIG_LIBOPENJPEG_DECODER)        += libopenjpegdec.o
diff --git a/libavcodec/libopenh264.c b/libavcodec/libopenh264.c
index 59c61a3a4c..14d83e27df 100644
--- a/libavcodec/libopenh264.c
+++ b/libavcodec/libopenh264.c
@@ -20,8 +20,13 @@
  */
 
 #include <string.h>
+
+#ifdef CONFIG_LIBOPENH264_DLOPEN
+#include "libopenh264_dlopen.h"
+#else
 #include <wels/codec_api.h>
 #include <wels/codec_ver.h>
+#endif
 
 #include "libavutil/log.h"
 
@@ -51,8 +56,17 @@ int ff_libopenh264_check_version(void *logctx)
     // Mingw GCC < 4.7 on x86_32 uses an incorrect/buggy ABI for the WelsGetCodecVersion
     // function (for functions returning larger structs), thus skip the check in those
     // configurations.
-#if !defined(_WIN32) || !defined(__GNUC__) || !ARCH_X86_32 || AV_GCC_VERSION_AT_LEAST(4, 7)
-    OpenH264Version libver = WelsGetCodecVersion();
+    // Also, for dlopened OpenH264, we should not do the version check. It's too punitive.
+#if !defined(_WIN32) || !defined(__GNUC__) || !ARCH_X86_32 || AV_GCC_VERSION_AT_LEAST(4, 7) || !defined(CONFIG_LIBOPENH264_DLOPEN)
+    OpenH264Version libver;
+
+#ifdef CONFIG_LIBOPENH264_DLOPEN
+    if (loadLibOpenH264(logctx)) {
+         return AVERROR_EXTERNAL;
+    }
+#endif
+
+    libver = WelsGetCodecVersion();
     if (memcmp(&libver, &g_stCodecVersion, sizeof(libver))) {
         av_log(logctx, AV_LOG_ERROR, "Incorrect library version loaded\n");
         return AVERROR(EINVAL);
diff --git a/libavcodec/libopenh264_dlopen.c b/libavcodec/libopenh264_dlopen.c
new file mode 100644
index 0000000000..49ea8ff44f
--- /dev/null
+++ b/libavcodec/libopenh264_dlopen.c
@@ -0,0 +1,147 @@
+/*
+ * OpenH264 dlopen code
+ *
+ * Copyright (C) 2022 Andreas Schneider <asn@cryptomilk.org>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <dlfcn.h>
+
+#include "libopenh264_dlopen.h"
+
+/*
+ * The symbol binding makes sure we do not run into strict aliasing issues which
+ * can lead into segfaults.
+ */
+typedef int (*__oh264_WelsCreateSVCEncoder)(ISVCEncoder **);
+typedef void (*__oh264_WelsDestroySVCEncoder)(ISVCEncoder *);
+typedef int (*__oh264_WelsGetDecoderCapability)(SDecoderCapability *);
+typedef long (*__oh264_WelsCreateDecoder)(ISVCDecoder **);
+typedef void (*__oh264_WelsDestroyDecoder)(ISVCDecoder *);
+typedef OpenH264Version (*__oh264_WelsGetCodecVersion)(void);
+typedef void (*__oh264_WelsGetCodecVersionEx)(OpenH264Version *);
+
+#define OH264_SYMBOL_ENTRY(i)                                                  \
+  union {                                                                      \
+    __oh264_##i f;                                                             \
+    void *obj;                                                                 \
+  } _oh264_##i
+
+struct oh264_symbols {
+  OH264_SYMBOL_ENTRY(WelsCreateSVCEncoder);
+  OH264_SYMBOL_ENTRY(WelsDestroySVCEncoder);
+  OH264_SYMBOL_ENTRY(WelsGetDecoderCapability);
+  OH264_SYMBOL_ENTRY(WelsCreateDecoder);
+  OH264_SYMBOL_ENTRY(WelsDestroyDecoder);
+  OH264_SYMBOL_ENTRY(WelsGetCodecVersion);
+  OH264_SYMBOL_ENTRY(WelsGetCodecVersionEx);
+};
+
+/* Symbols are bound by loadLibOpenH264() */
+static struct oh264_symbols openh264_symbols;
+
+int oh264_WelsCreateSVCEncoder(ISVCEncoder **ppEncoder) {
+  return openh264_symbols._oh264_WelsCreateSVCEncoder.f(ppEncoder);
+}
+
+void oh264_WelsDestroySVCEncoder(ISVCEncoder *pEncoder) {
+  return openh264_symbols._oh264_WelsDestroySVCEncoder.f(pEncoder);
+}
+
+int oh264_WelsGetDecoderCapability(SDecoderCapability *pDecCapability) {
+  return openh264_symbols._oh264_WelsGetDecoderCapability.f(pDecCapability);
+}
+
+long oh264_WelsCreateDecoder(ISVCDecoder **ppDecoder) {
+  return openh264_symbols._oh264_WelsCreateDecoder.f(ppDecoder);
+}
+
+void oh264_WelsDestroyDecoder(ISVCDecoder *pDecoder) {
+  return openh264_symbols._oh264_WelsDestroyDecoder.f(pDecoder);
+}
+
+OpenH264Version oh264_WelsGetCodecVersion(void) {
+  return openh264_symbols._oh264_WelsGetCodecVersion.f();
+}
+
+void oh264_WelsGetCodecVersionEx(OpenH264Version *pVersion) {
+  openh264_symbols._oh264_WelsGetCodecVersionEx.f(pVersion);
+}
+
+static void *_oh264_bind_symbol(AVCodecContext *avctx,
+                                void *handle,
+                                const char *sym_name) {
+    void *sym = NULL;
+
+    sym = dlsym(handle, sym_name);
+    if (sym == NULL) {
+        const char *err = dlerror();
+        av_log(avctx,
+               AV_LOG_WARNING,
+               "%s: Failed to bind %s\n",
+               err,
+               sym_name);
+        return NULL;
+    }
+
+    return sym;
+}
+
+#define oh264_bind_symbol(avctx, handle, sym_name)                           \
+  if (openh264_symbols._oh264_##sym_name.obj == NULL) {                      \
+    openh264_symbols._oh264_##sym_name.obj = _oh264_bind_symbol(avctx, handle, #sym_name); \
+    if (openh264_symbols._oh264_##sym_name.obj == NULL) {                    \
+      return 1;                                                              \
+    }                                                                        \
+  }
+
+int loadLibOpenH264(AVCodecContext *avctx) {
+  static bool initialized = false;
+  void *libopenh264 = NULL;
+  const char *err = NULL;
+
+  if (initialized) {
+      return 0;
+  }
+
+#define OPENH264_LIB "libopenh264.so.7"
+  libopenh264 = dlopen(OPENH264_LIB, RTLD_LAZY);
+  err = dlerror();
+  if (err != NULL) {
+    av_log(avctx, AV_LOG_WARNING,
+           "%s: %s is missing, openh264 support will be disabled\n", err,
+           OPENH264_LIB);
+
+    if (libopenh264 != NULL) {
+      dlclose(libopenh264);
+    }
+    return 1;
+  }
+
+  oh264_bind_symbol(avctx, libopenh264, WelsCreateSVCEncoder);
+  oh264_bind_symbol(avctx, libopenh264, WelsDestroySVCEncoder);
+  oh264_bind_symbol(avctx, libopenh264, WelsGetDecoderCapability);
+  oh264_bind_symbol(avctx, libopenh264, WelsCreateDecoder);
+  oh264_bind_symbol(avctx, libopenh264, WelsDestroyDecoder);
+  oh264_bind_symbol(avctx, libopenh264, WelsGetCodecVersion);
+  oh264_bind_symbol(avctx, libopenh264, WelsGetCodecVersionEx);
+
+  initialized = true;
+
+  return 0;
+}
diff --git a/libavcodec/libopenh264_dlopen.h b/libavcodec/libopenh264_dlopen.h
new file mode 100644
index 0000000000..d7d8bb7cad
--- /dev/null
+++ b/libavcodec/libopenh264_dlopen.h
@@ -0,0 +1,58 @@
+/*
+ * OpenH264 dlopen code
+ *
+ * Copyright (C) 2022 Andreas Schneider <asn@cryptomilk.org>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef HAVE_LIBOPENH264_DLOPEN_H
+#define HAVE_LIBOPENH264_DLOPEN_H
+
+#ifdef CONFIG_LIBOPENH264_DLOPEN
+
+#include <wels/codec_api.h>
+#include <wels/codec_ver.h>
+
+#include "avcodec.h"
+
+int oh264_WelsCreateSVCEncoder(ISVCEncoder **ppEncoder);
+#define WelsCreateSVCEncoder oh264_WelsCreateSVCEncoder
+
+void oh264_WelsDestroySVCEncoder(ISVCEncoder *pEncoder);
+#define WelsDestroySVCEncoder oh264_WelsDestroySVCEncoder
+
+int oh264_WelsGetDecoderCapability(SDecoderCapability *pDecCapability);
+#define WelsGetDecoderCapability oh264_WelsGetDecoderCapability
+
+long oh264_WelsCreateDecoder(ISVCDecoder **ppDecoder);
+#define WelsCreateDecoder oh264_WelsCreateDecoder
+
+void oh264_WelsDestroyDecoder(ISVCDecoder *pDecoder);
+#define WelsDestroyDecoder oh264_WelsDestroyDecoder
+
+OpenH264Version oh264_WelsGetCodecVersion(void);
+#define WelsGetCodecVersion oh264_WelsGetCodecVersion
+
+void oh264_WelsGetCodecVersionEx(OpenH264Version *pVersion);
+#define WelsGetCodecVersionEx oh264_WelsGetCodecVersionEx
+
+int loadLibOpenH264(AVCodecContext *avctx);
+
+#endif /* CONFIG_LIBOPENH264_DLOPEN */
+
+#endif /* HAVE_LIBOPENH264_DLOPEN_H */
diff --git a/libavcodec/libopenh264dec.c b/libavcodec/libopenh264dec.c
index 97d3630df6..feb0a0927e 100644
--- a/libavcodec/libopenh264dec.c
+++ b/libavcodec/libopenh264dec.c
@@ -19,8 +19,12 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#ifdef CONFIG_LIBOPENH264_DLOPEN
+#include "libopenh264_dlopen.h"
+#else
 #include <wels/codec_api.h>
 #include <wels/codec_ver.h>
+#endif
 
 #include "libavutil/common.h"
 #include "libavutil/fifo.h"
@@ -55,6 +59,12 @@ static av_cold int svc_decode_init(AVCodecContext *avctx)
     int log_level;
     WelsTraceCallback callback_function;
 
+#ifdef CONFIG_LIBOPENH264_DLOPEN
+    if (loadLibOpenH264(avctx)) {
+         return AVERROR_DECODER_NOT_FOUND;
+    }
+#endif
+
     if ((err = ff_libopenh264_check_version(avctx)) < 0)
         return AVERROR_DECODER_NOT_FOUND;
 
diff --git a/libavcodec/libopenh264enc.c b/libavcodec/libopenh264enc.c
index 2001dc22b4..c8b1348df2 100644
--- a/libavcodec/libopenh264enc.c
+++ b/libavcodec/libopenh264enc.c
@@ -19,8 +19,12 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#ifdef CONFIG_LIBOPENH264_DLOPEN
+#include "libopenh264_dlopen.h"
+#else
 #include <wels/codec_api.h>
 #include <wels/codec_ver.h>
+#endif
 
 #include "libavutil/attributes.h"
 #include "libavutil/common.h"
@@ -136,6 +140,12 @@ static av_cold int svc_encode_init(AVCodecContext *avctx)
     WelsTraceCallback callback_function;
     AVCPBProperties *props;
 
+#ifdef CONFIG_LIBOPENH264_DLOPEN
+    if (loadLibOpenH264(avctx)) {
+         return AVERROR_ENCODER_NOT_FOUND;
+    }
+#endif
+
     if ((err = ff_libopenh264_check_version(avctx)) < 0)
         return AVERROR_ENCODER_NOT_FOUND;
 
-- 
2.36.1