Move vendoring of Fluent-Bit to CMake.

.gitmodules

@@ -10,8 +10,3 @@
 	path = src/web/server/h2o/libh2o
 	url = https://github.com/h2o/h2o.git
 	ignore = untracked
-[submodule "fluent-bit"]
-	path = src/fluent-bit
-	url = https://github.com/fluent/fluent-bit.git
-	shallow = true
-	ignore = dirty

CMakeLists.txt

@@ -275,6 +275,10 @@ if(ENABLE_LEGACY_EBPF_PROGRAMS)
         include(NetdataEBPFLegacy)
 endif()
 
+if(ENABLE_PLUGIN_LOGS_MANAGEMENT)
+        include(NetdataFluentBit)
+endif()
+
 if(ENABLE_SENTRY)
         include(NetdataSentry)
 endif()
@@ -290,6 +294,10 @@ if(ENABLE_LEGACY_EBPF_PROGRAMS)
         netdata_fetch_legacy_ebpf_code()
 endif()
 
+if(ENABLE_PLUGIN_LOGS_MANAGEMENT)
+        netdata_bundle_flb()
+endif()
+
 if(ENABLE_SENTRY)
         netdata_bundle_sentry()
 endif()
@@ -1921,6 +1929,8 @@ if(ENABLE_PLUGIN_LOGS_MANAGEMENT)
     add_executable(logs-management.plugin ${LOGS_MANAGEMENT_PLUGIN_FILES})
     target_link_libraries(logs-management.plugin libnetdata)
 
+    netdata_require_flb_for_target(logs-management.plugin)
+
     install(TARGETS logs-management.plugin
             COMPONENT plugin-logs-management
             DESTINATION usr/libexec/netdata/plugins.d)
@@ -1947,6 +1957,10 @@ if(ENABLE_PLUGIN_LOGS_MANAGEMENT)
                 COMPONENT plugin-logs-management
                 DESTINATION usr/share/doc/netdata-plugin-logs-management)
     endif()
+
+    install(DIRECTORY
+            COMPONENT plugin-logs-management
+            DESTINATION etc/netdata/logsmanagement.d)
 endif()
 
 if(ENABLE_PLUGIN_CUPS)

netdata-installer.sh

@@ -272,7 +272,7 @@
     "--zlib-is-really-here") LIBS_ARE_HERE=1 ;;
     "--libs-are-really-here") LIBS_ARE_HERE=1 ;;
     "--use-system-protobuf") USE_SYSTEM_PROTOBUF=1 ;;
     "--dont-scrub-cflags-even-though-it-may-break-things") DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS=1 ;;
     "--dont-start-it") DONOTSTART=1 ;;
     "--dont-wait") DONOTWAIT=1 ;;
     "--auto-update" | "-u") ;;
@@ -555,78 +555,6 @@
 
 trap build_error EXIT
 
-# -----------------------------------------------------------------------------
-build_fluentbit() {
-  env_cmd="env CFLAGS='-w' CXXFLAGS='-w' LDFLAGS="
-
-  if [ -z "${DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS}" ]; then
-    env_cmd="env CFLAGS='-fPIC -pipe -w' CXXFLAGS='-fPIC -pipe -w' LDFLAGS="
-  fi
-
-  mkdir -p src/fluent-bit/build || return 1
-  cd src/fluent-bit/build > /dev/null || return 1
-
-  rm CMakeCache.txt > /dev/null 2>&1
-
-  if ! run eval "${env_cmd} $1 -C ../../logsmanagement/fluent_bit_build/config.cmake -B./ -S../"; then
-    cd - > /dev/null || return 1
-    rm -rf src/fluent-bit/build > /dev/null 2>&1
-    return 1
-  fi
-
-  if ! run eval "${env_cmd} ${make} ${MAKEOPTS}"; then
-    cd - > /dev/null || return 1
-    rm -rf src/fluent-bit/build > /dev/null 2>&1
-    return 1
-  fi
-
-  cd - > /dev/null || return 1
-}
-
-bundle_fluentbit() {
-  progress "Prepare Fluent-Bit"
-
-  if [ "${ENABLE_LOGS_MANAGEMENT}" = 0 ]; then
-    warning "You have explicitly requested to disable Netdata Logs Management support, Fluent-Bit build is skipped."
-    return 0
-  fi
-
-  if [ ! -d "src/fluent-bit" ]; then
-    warning "Missing submodule Fluent-Bit. The install process will continue, but Netdata Logs Management support will be disabled."
-    ENABLE_LOGS_MANAGEMENT=0
-    return 0
-  fi
-
-  patch -N -p1 src/fluent-bit/CMakeLists.txt -i src/logsmanagement/fluent_bit_build/CMakeLists.patch
-  patch -N -p1 src/fluent-bit/src/flb_log.c -i src/logsmanagement/fluent_bit_build/flb-log-fmt.patch
-
-  # If musl is used, we need to patch chunkio, providing fts has been previously installed.
-  libc="$(detect_libc)"
-  if [ "${libc}" = "musl" ]; then
-    patch -N -p1 src/fluent-bit/lib/chunkio/src/CMakeLists.txt -i src/logsmanagement/fluent_bit_build/chunkio-static-lib-fts.patch
-    patch -N -p1 src/fluent-bit/cmake/luajit.cmake -i src/logsmanagement/fluent_bit_build/exclude-luajit.patch
-    patch -N -p1 src/fluent-bit/src/flb_network.c -i src/logsmanagement/fluent_bit_build/xsi-strerror.patch
-  fi
-
-  [ -n "${GITHUB_ACTIONS}" ] && echo "::group::Bundling Fluent-Bit."
-
-  if build_fluentbit "$cmake"; then
-    # If Fluent-Bit built with inotify support, use it.
-    if [ "$(grep -o '^FLB_HAVE_INOTIFY:INTERNAL=.*' src/fluent-bit/build/CMakeCache.txt | cut -d '=' -f 2)" ]; then
-      CFLAGS="${CFLAGS} -DFLB_HAVE_INOTIFY"
-    fi
-    FLUENT_BIT_BUILD_SUCCESS=1
-    run_ok "Fluent-Bit built successfully."
-  else
-    warning "Failed to build Fluent-Bit, Netdata Logs Management support will be disabled in this build."
-    ENABLE_LOGS_MANAGEMENT=0
-  fi
-
-  [ -n "${GITHUB_ACTIONS}" ] && echo "::endgroup::"
-}
-
-bundle_fluentbit
-
 # -----------------------------------------------------------------------------
 # If we’re installing the Go plugin, ensure a working Go toolchain is installed.
 if [ "${ENABLE_GO}" -eq 1 ]; then
@@ -1055,70 +983,6 @@
 
 [ -n "${GITHUB_ACTIONS}" ] && echo "::endgroup::"
 
-detect_libc() {
-  libc=
-  if ldd --version 2>&1 | grep -q -i glibc; then
-    echo >&2 " Detected GLIBC"
-    libc="glibc"
-  elif ldd --version 2>&1 | grep -q -i 'gnu libc'; then
-    echo >&2 " Detected GLIBC"
-    libc="glibc"
-  elif ldd --version 2>&1 | grep -q -i musl; then
-    echo >&2 " Detected musl"
-    libc="musl"
-  else
-      cmd=$(ldd /bin/sh | grep -w libc | cut -d" " -f 3)
-      if bash -c "${cmd}" 2>&1 | grep -q -i "GNU C Library"; then
-        echo >&2 " Detected GLIBC"
-        libc="glibc"
-      fi
-  fi
-
-  if [ -z "$libc" ]; then
-    warning "Cannot detect a supported libc on your system, eBPF support will be disabled."
-    return 1
-  fi
-
-  echo "${libc}"
-}
-
-should_install_fluentbit() {
-  if [ "$(uname -s)" = "Darwin" ]; then
-    return 1
-  fi
-  if [ "${ENABLE_LOGS_MANAGEMENT}" = 0 ]; then
-    warning "netdata-installer.sh run with --disable-logsmanagement, Fluent-Bit installation is skipped."
-    return 1
-  elif [ "${FLUENT_BIT_BUILD_SUCCESS:=0}" -eq 0 ]; then
-    run_failed "Fluent-Bit was not built successfully, Netdata Logs Management support will be disabled in this build."
-    return 1
-  elif [ ! -f src/fluent-bit/build/lib/libfluent-bit.so ]; then
-    run_failed "libfluent-bit.so is missing, Netdata Logs Management support will be disabled in this build."
-    return 1
-  fi
-
-  return 0
-}
-
-install_fluentbit() {
-  if ! should_install_fluentbit; then
-    enable_feature PLUGIN_LOGS_MANAGEMENT 0
-    return 0
-  fi
-
-  [ -n "${GITHUB_ACTIONS}" ] && echo "::group::Installing Fluent-Bit."
-
-  run chown "root:${NETDATA_GROUP}" src/fluent-bit/build/lib
-  run chmod 0644 src/fluent-bit/build/lib/libfluent-bit.so
-
-  run cp -a -v src/fluent-bit/build/lib/libfluent-bit.so "${NETDATA_PREFIX}"/usr/lib/netdata
-
-  [ -n "${GITHUB_ACTIONS}" ] && echo "::endgroup::"
-}
-
-progress "Installing Fluent-Bit plugin"
-install_fluentbit
-
 # -----------------------------------------------------------------------------
 progress "Telemetry configuration"
 

netdata.spec.in

@@ -607,6 +607,7 @@ rm -rf "${RPM_BUILD_ROOT}"
 
 # logs management belongs to a different sub-package
 %exclude %{_libexecdir}/%{name}/plugins.d/logs-management.plugin
+%exclude %{_libdir}/%{name}/libfluent-bit.so
 %exclude %{_libdir}/%{name}/conf.d/logsmanagement.d.conf
 %exclude %{_libdir}/%{name}/conf.d/logsmanagement.d
 
@@ -969,6 +970,7 @@ fi
 %defattr(0644,root,netdata,0755)
 %{_libdir}/%{name}/conf.d/logsmanagement.d.conf
 %{_libdir}/%{name}/conf.d/logsmanagement.d
+%{_libdir}/%{name}/libfluent-bit.so
 %defattr(0750,root,netdata,0750)
 # CAP_DAC_READ_SEARCH and CAP_SYSLOG needed for data collection.
 %caps(cap_dac_read_search,cap_syslog=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/logs-management.plugin

packaging/build-package.sh

@@ -38,7 +38,7 @@ add_cmake_option ENABLE_PLUGIN_DEBUGFS On
 add_cmake_option ENABLE_PLUGIN_FREEIPMI On
 add_cmake_option ENABLE_PLUGIN_GO On
 add_cmake_option ENABLE_PLUGIN_LOCAL_LISTENERS On
-add_cmake_option ENABLE_PLUGIN_LOGS_MANAGEMENT Off
+add_cmake_option ENABLE_PLUGIN_LOGS_MANAGEMENT On
 add_cmake_option ENABLE_PLUGIN_NFACCT On
 add_cmake_option ENABLE_PLUGIN_PERF On
 add_cmake_option ENABLE_PLUGIN_SLABINFO On

packaging/cmake/Modules/NetdataFluentBit.cmake

@@ -0,0 +1,65 @@
+# Macros and functions for handling of Fluent-Bit
+#
+# Copyright (c) 2024 Netdata Inc.
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set(_ND_FLB_SRC_DIR "${CMAKE_BINARY_DIR}/_deps/fluentbit-src")
+
+# Handle bundling of fluent-bit
+function(netdata_bundle_flb)
+    include(FetchContent)
+    include(NetdataFetchContentExtra)
+    include(NetdataUtil)
+
+    find_program(PATCH patch REQUIRED)
+
+    set(_need_musl_patches 0)
+
+    netdata_identify_libc(_libc_name)
+
+    if(_libc_name MATCHES "musl")
+        set(_need_musl_patches 1)
+    endif()
+
+    message(STATUS "Preparing vendored copy of fluent-bit")
+
+    set(FETCHCONTENT_TRY_FIND_PACKAGE_MODE NEVER)
+
+    include("${CMAKE_SOURCE_DIR}/src/logsmanagement/fluent_bit_build/config.cmake")
+
+    FetchContent_Declare(
+        fluentbit
+        GIT_REPOSITORY https://github.com/fluent/fluent-bit.git
+        GIT_TAG b19e9ce674de872640c00a697fa545b66df0628a # last used submodule commit
+        GIT_PROGRESS On
+        SOURCE_DIR ${_ND_FLB_SRC_DIR}
+        PATCH_COMMAND ${CMAKE_SOURCE_DIR}/src/logsmanagement/fluent_bit_build/apply-patches.sh ${_ND_FLB_SRC_DIR} ${_need_musl_patches}
+        CMAKE_ARGS ${NETDATA_CMAKE_PROPAGATE_TOOLCHAIN_ARGS}
+    )
+
+    FetchContent_MakeAvailable_NoInstall(fluentbit)
+
+    add_custom_command(
+        OUTPUT ${PROJECT_BINARY_DIR}/include/fluent-bit
+        COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/include
+        COMMAND ${CMAKE_COMMAND} -E create_symlink ${_ND_FLB_SRC_DIR} ${PROJECT_BINARY_DIR}/include/fluent-bit
+        COMMENT "Create symlink for vendored Fluent-BIt headers"
+        DEPENDS fluent-bit-shared
+    )
+    add_custom_target(
+        flb-include-link
+        DEPENDS ${PROJECT_BINARY_DIR}/include/fluent-bit
+    )
+
+    install(FILES ${fluentbit_BINARY_DIR}/lib/libfluent-bit.so
+            DESTINATION usr/lib/netdata
+            COMPONENT plugin-logs-management)
+
+    message(STATUS "Finished preparing vendored copy of fluent-bit")
+endfunction()
+
+# Ensure that fluentbit gets built if the specified target is built
+function(netdata_require_flb_for_target _target)
+    target_include_directories(${_target} PUBLIC "${PROJECT_BINARY_DIR}/include")
+    add_dependencies(${_target} flb-include-link)
+endfunction()

packaging/cmake/config.cmake.h.in

@@ -117,6 +117,8 @@
 
 #cmakedefine ENABLE_LOGSMANAGEMENT
 #cmakedefine ENABLE_LOGSMANAGEMENT_TESTS
+#cmakedefine FLB_HAVE_INOTIFY
+
 #cmakedefine ENABLE_PROMETHEUS_REMOTE_WRITE
 
 // enabled plugins

packaging/installer/install-required-packages.sh

@@ -671,14 +671,18 @@ declare -A pkg_automake=(
   ['default']="automake"
 )
 
-# Required to build libwebsockets and libmosquitto on some systems.
 declare -A pkg_cmake=(
   ['gentoo']="dev-util/cmake"
   ['clearlinux']="c-basic"
   ['default']="cmake"
 )
 
-# bison and flex are required by Fluent-Bit
+# patch, bison and flex are required by Fluent-Bit
+declare -A pkg_patch=(
+  ['default']="patch"
+  ['gentoo']="dev-util/patch"
+)
+
 declare -A pkg_bison=(
   ['default']="bison"
 )
@@ -690,7 +694,7 @@ declare -A pkg_flex=(
 # fts-dev is required by Fluent-Bit on Alpine
 declare -A pkg_fts_dev=(
   ['default']="NOTREQUIRED"
-  ['alpine']="musl-fts-dev" 
+  ['alpine']="musl-fts-dev"
   ['alpine-3.16.9']="fts-dev"
 )
 
@@ -1279,6 +1283,7 @@ packages() {
     require_cmd tar || suitable_package tar
     require_cmd curl || suitable_package curl
     require_cmd gzip || suitable_package gzip
+    require_cmd patch || suitable_package patch
     require_cmd bison || suitable_package bison
     require_cmd flex || suitable_package flex
   fi

src/logsmanagement/flb_plugin.c

@@ -12,9 +12,9 @@
 #include "circular_buffer.h"
 #include "daemon/common.h"
 #include "libnetdata/libnetdata.h"
-#include "../fluent-bit/lib/msgpack-c/include/msgpack/unpack.h"
-#include "../fluent-bit/lib/msgpack-c/include/msgpack/object.h"
-#include "../fluent-bit/lib/monkey/include/monkey/mk_core/mk_list.h"
+#include "fluent-bit/lib/msgpack-c/include/msgpack/unpack.h"
+#include "fluent-bit/lib/msgpack-c/include/msgpack/object.h"
+#include "fluent-bit/lib/monkey/include/monkey/mk_core/mk_list.h"
 #include <dlfcn.h>
 
 #ifdef HAVE_SYSTEMD
@@ -1533,4 +1533,4 @@ int flb_add_fwd_input(Flb_socket_config_t *forward_in_config){
 
 void flb_free_fwd_input_out_cb(void){
     freez(fwd_input_out_cb);
-}
+}

src/logsmanagement/fluent_bit_build/apply-patches.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+FLUENT_BIT_SRC="${1}"
+NEED_MUSL_FTS_PATCHES="${2}"
+
+SCRIPT_SOURCE="$(
+    self=${0}
+    while [ -L "${self}" ]
+    do
+        cd "${self%/*}" || exit 1
+        self=$(readlink "${self}")
+    done
+    cd "${self%/*}" || exit 1
+    echo "$(pwd -P)/${self##*/}"
+)"
+PATCH_DIR="$(dirname "${SCRIPT_SOURCE}")"
+
+cd "${FLUENT_BIT_SRC}" || exit 1
+
+patch -p1 -i "${PATCH_DIR}/CMakeLists.patch" || exit 1
+patch -p1 -i "${PATCH_DIR}/flb-log-fmt.patch" || exit 1
+patch -p1 -i "${PATCH_DIR}/flb-ninja.patch" || exit 1
+
+if [ "${NEED_MUSL_FTS_PATCHES:-0}" -eq 1 ]; then
+    patch -p1 -i "${PATCH_DIR}/chunkio-static-lib-fts.patch" || exit 1
+    patch -p1 -i "${PATCH_DIR}/exclude-luajit.patch" || exit 1
+    patch -p1 -i "${PATCH_DIR}/xsi-strerror.patch" || exit 1
+fi