UI: Show warning on startup if selected encoder is missing

UI/data/locale/en-US.ini

@@ -1589,3 +1589,30 @@ MultitrackVideo.IncompatibleSettings.UpdateAndStartStreaming="Update Settings an
 MultitrackVideo.IncompatibleSettings.AudioChannels="%1 is not currently compatible with [Audio → General → Channels] set to '%2', %3"
 MultitrackVideo.IncompatibleSettings.AudioChannelsSingle="[Audio → General → Channels] needs to be set to '%1'"
 MultitrackVideo.IncompatibleSettings.AudioChannelsMultiple="%1 requires multiple different settings for [Audio → General → Channels]"
+
+# Missing Encoder Warning
+EncoderMissing.Title="Missing Encoders"
+EncoderMissing.Text="The following configured encoders are missing:\n<ul>\n%1</ul><br>\nSee the <a href=\"%2\">Knowledge Base Article</a> for further information."
+EncoderMissing.ItemText="<li>\n<h3>%1</h3>\n%2 (code: <code>%3</code>)\n</li>"
+EncoderMissing.Unknown="Unknown"
+EncoderMissing.MissingModule="Module \"%1\" not loaded"
+
+## Nvenc specific errors
+EncoderMissing.NVENC.TestProgramFailedStartup="NVENC check program start failure"
+EncoderMissing.NVENC.TestProgramExitWithError="NVENC check process failure, exit code: %1"
+EncoderMissing.NVENC.TestProgramReadFailure="NVENC check process output read failure, exit code: %1"
+EncoderMissing.NVENC.TestProgramError="NVENC check failed with reason: %1"
+EncoderMissing.NVENC.NoDevices="No NVIDIA GPUs found"
+EncoderMissing.NVENC.Unsupported.Kepler="The architecture (Kepler) of your GPU (%1) is no longer supported. Please use OBS 30.2 or earlier instead."
+EncoderMissing.NVENC.Reason.NvmlLoad="Failed loading NVML library"
+EncoderMissing.NVENC.Reason.NvmlInit="Failed initializing NVML"
+EncoderMissing.NVENC.Reason.NvencLoad="Failed loading NVENC library"
+EncoderMissing.NVENC.Reason.NvencInit="Failed initializing NVENC"
+EncoderMissing.NVENC.Reason.NvencVersion="NVENC version empty"
+EncoderMissing.NVENC.Reason.CudaLoad="Failed loading CUDA library"
+EncoderMissing.NVENC.Reason.CudaInit="Failed initializing CUDA"
+EncoderMissing.NVENC.Reason.CudaVersion="Invalid/Missing CUDA version"
+EncoderMissing.NVENC.Reason.DriverOutdated="Outdated driver"
+EncoderMissing.NVENC.Reason.NoSupportedDevices="No NVIDIA devices with NVENC support found"
+EncoderMissing.NVENC.Reason.SessionLimitExceeded="Encoder session limit exceeded"
+EncoderMissing.NVENC.Reason.AV1Unsupported="AV1 encoding is not supported by any of the system's NVIDIA GPUs"

UI/window-basic-main-profiles.cpp

@@ -20,13 +20,16 @@
 #include <string>
 #include <map>
 #include <tuple>
+#include <unordered_set>
 #include <obs.hpp>
+#include <util/pipe.h>
 #include <util/platform.h>
 #include <util/util.hpp>
 #include <QMessageBox>
 #include <QVariant>
 #include <QFileDialog>
 #include <qt-wrappers.hpp>
+
 #include "window-basic-main.hpp"
 #include "window-basic-auto-config.hpp"
 #include "window-namedialog.hpp"
@@ -736,6 +739,7 @@ void OBSBasic::ActivateProfile(const OBSProfile &profile, bool reset)
 void OBSBasic::UpdateProfileEncoders()
 {
 	InitBasicConfigDefaults2();
+	CheckForMissingEncoders();
 	CheckForSimpleModeX264Fallback();
 }
 
@@ -906,3 +910,174 @@ void OBSBasic::CheckForSimpleModeX264Fallback()
 		activeConfiguration.SaveSafe("tmp");
 	}
 }
+
+static auto args_deleter = [](os_process_args_t *args) {
+	os_process_args_destroy(args);
+};
+using OsProcessArgs = std::unique_ptr<os_process_args_t, decltype(args_deleter)>;
+using CheckResult = std::optional<std::pair<QString, QString>>;
+
+#ifndef __APPLE__
+static CheckResult CheckNVENCInternal(std::unordered_set<std::string_view> &missing_encoders)
+{
+	const std::vector<std::pair<std::string_view, std::string_view>> nvencFailureReasons = {
+		{"nvml_lib", "EncoderMissing.NVENC.Reason.NvmlLoad"},
+		// Some error messages are a prefix + numeric error code
+		{"nvml_init", "EncoderMissing.NVENC.Reason.NvmlInit"},
+		{"nvenc_lib", "EncoderMissing.NVENC.Reason.NvencLoad"},
+		{"nvenc_init", "EncoderMissing.NVENC.Reason.NvencInit"},
+		{"cuda_lib", "EncoderMissing.NVENC.Reason.CudaLoad"},
+		{"cuda_init", "EncoderMissing.NVENC.Reason.CudaInit"},
+		{"no_cuda_version", "EncoderMissing.NVENC.Reason.CudaVersion"},
+		{"no_devices", "EncoderMissing.NVENC.NoDevices"},
+		{"no_nvenc_version", "EncoderMissing.NVENC.Reason.NvencVersion"},
+		{"outdated_driver", "EncoderMissing.NVENC.Reason.DriverOutdated"},
+		{"no_supported_devices", "EncoderMissing.NVENC.Reason.NoSupportedDevices"},
+		{"session_limit", "EncoderMissing.NVENC.Reason.SessionLimitExceeded"},
+	};
+	const bool includes_av1 = missing_encoders.count("obs_nvenc_av1_cuda") ||
+				  missing_encoders.count("obs_nvenc_av1_soft") ||
+				  missing_encoders.count("obs_nvenc_av1_tex");
+
+#ifdef _WIN32
+	BPtr test_exe = os_get_executable_path_ptr("obs-nvenc-test.exe");
+#else
+	BPtr test_exe = os_get_executable_path_ptr("obs-nvenc-test");
+#endif
+
+	OsProcessArgs args{os_process_args_create(test_exe), args_deleter};
+
+	os_process_pipe_t *pp = os_process_pipe_create2(args.get(), "r");
+	if (!pp)
+		return std::make_pair("", QTStr("EncoderMissing.NVENC.TestProgramFailedStartup"));
+
+	std::string caps_result;
+	for (;;) {
+		char data[2048];
+		size_t len = os_process_pipe_read(pp, (uint8_t *)data, sizeof(data));
+		if (!len)
+			break;
+
+		caps_result.append(data, len);
+	}
+
+	int exit_code = os_process_pipe_destroy(pp);
+
+	if (caps_result.empty())
+		return std::make_pair("", QTStr("EncoderMissing.NVENC.TestProgramExitWithError").arg(exit_code));
+
+	auto config = ConfigFile();
+	if (config.OpenString(caps_result.c_str()) != CONFIG_SUCCESS)
+		return std::make_pair("", QTStr("EncoderMissing.NVENC.TestProgramReadFailure").arg(exit_code));
+
+	/* Read devices and attempt to find reason for failure */
+	auto numDevices = config_get_uint(config, "general", "cuda_devices");
+	if (!numDevices)
+		return std::make_pair("", QTStr("EncoderMissing.NVENC.NoDevices"));
+
+	/* Check device generation (architecture) */
+	for (uint64_t i = 0; i < numDevices; i++) {
+		std::string section = "device." + std::to_string(i);
+
+		if (!config_has_user_value(config, section.c_str(), "name"))
+			continue;
+
+		const QString name(config_get_string(config, section.c_str(), "name"));
+		const std::string_view arch = config_get_string(config, section.c_str(), "architecture_name");
+
+		if (arch == "Kepler")
+			return std::make_pair("kepler", QTStr("EncoderMissing.NVENC.Unsupported.Kepler").arg(name));
+	}
+
+	/* All other specific failures of the test binary */
+	if (config_has_user_value(config, "general", "reason")) {
+		const std::string_view reason = config_get_string(config, "general", "reason");
+
+		for (auto &[code, desc] : nvencFailureReasons) {
+			if (reason.find(code) == std::string_view::npos)
+				continue;
+
+			QString ret = QTStr("EncoderMissing.NVENC.TestProgramError").arg(QTStr(desc.data()));
+			QString crumb = QString::fromUtf8(code.data());
+			return std::make_pair(crumb, ret);
+		}
+	}
+
+	/* Finally, if everything else looks good but the encoder missing is AV1, and AV1 is not supported on any GPUs,
+	 * that's probably the issue the user is having (for example, when copying a profile from another machine). */
+	if (includes_av1 && !config_get_bool(config, "av1", "codec_supported"))
+		return std::make_pair("no_av1", QTStr("EncoderMissing.NVENC.Reason.AV1Unsupported"));
+
+	return std::nullopt;
+}
+
+static CheckResult CheckNVENC(std::unordered_set<std::string_view> &missing_encoders)
+{
+	static auto result = CheckNVENCInternal(missing_encoders);
+	return result;
+}
+#endif
+
+static CheckResult CheckX264()
+{
+	static const bool module_loaded = obs_get_module("obs-x264") != nullptr;
+	/* This should be the only failure mode possible here. */
+	if (!module_loaded) {
+		return std::make_pair("plugin_not_loaded", QTStr("EncoderMissing.MissingModule").arg("obs-x264"));
+	}
+
+	return std::nullopt;
+}
+
+extern const char *get_simple_output_encoder(const char *name);
+
+void OBSBasic::CheckForMissingEncoders()
+{
+	constexpr QStringView kbURL(u"https://obsproject.com/kb/encoder-missing#%1");
+	const QString encoderListItem = QTStr("EncoderMissing.ItemText");
+	const QString unknownErrorText = QTStr("EncoderMissing.Unknown");
+
+	std::unordered_set<std::string_view> missing_encoders = {
+		config_get_string(activeConfiguration, "AdvOut", "Encoder"),
+		config_get_string(activeConfiguration, "AdvOut", "RecEncoder"),
+		get_simple_output_encoder(config_get_string(activeConfiguration, "SimpleOutput", "StreamEncoder")),
+		get_simple_output_encoder(config_get_string(activeConfiguration, "SimpleOutput", "RecEncoder")),
+	};
+
+	size_t idx = 0;
+	const char *id;
+	while (obs_enum_encoder_types(idx++, &id))
+		missing_encoders.erase(id);
+
+	if (missing_encoders.empty() || (missing_encoders.size() == 1 && missing_encoders.count("none")))
+		return;
+
+	QStringView crumb;
+	QStringList encoderList;
+
+	for (auto &encoder : missing_encoders) {
+		std::optional<std::pair<QStringView, QString>> res;
+
+		if (encoder.find("x264") != std::string_view::npos) {
+			res = CheckX264();
+		}
+#ifndef __APPLE__
+		else if (encoder.find("nvenc") != std::string_view::npos) {
+			res = CheckNVENC(missing_encoders);
+		}
+#endif
+
+		if (res) {
+			encoderList += encoderListItem.arg(encoder.data()).arg(res->second).arg(res->first);
+			if (crumb.empty())
+				crumb = res->first;
+		} else {
+			encoderList += encoderListItem.arg(encoder.data()).arg(unknownErrorText).arg("unknown");
+		}
+	}
+
+	// Format and show error message
+	const QString encoderMissingMessage =
+		QTStr("EncoderMissing.Text").arg(encoderList.join("\n")).arg(kbURL.arg(crumb));
+	OBSMessageBox::warning(this, QTStr("EncoderMissing.Title"), encoderMissingMessage, true);
+}

UI/window-basic-main.hpp

@@ -1305,6 +1305,7 @@ public slots:
 	std::vector<std::string> GetRestartRequirements(const ConfigFile &config) const;
 	void ResetProfileData();
 	void CheckForSimpleModeX264Fallback();
+	void CheckForMissingEncoders();
 
 public:
 	inline const OBSProfileCache &GetProfileCache() const noexcept { return profiles; };