Allow installation to close shell extension DLLs via the custom action. Disable reboot prompt in case of the version with this change was previously already installed.

CMakeLists.txt

@@ -23,8 +23,13 @@ set(BIN_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
 
 include(${CMAKE_SOURCE_DIR}/NEXTCLOUD.cmake)
 
+set ( NCCONTEXTMENU_SHELLEXT_WINDOW_CLASS_NAME "${APPLICATION_SHORTNAME}_Shxt_CntMenuHndlr_WndClass" )
+set ( NCOVERLAYS_SHELLEXT_WINDOW_CLASS_NAME "${APPLICATION_SHORTNAME}_Shxt_Ovs_WndClass" )
+
 # CfAPI Shell Extensions
 set( CFAPI_SHELL_EXTENSIONS_LIB_NAME CfApiShellExtensions )
+
+set ( CFAPI_SHELLEXT_WINDOW_CLASS_NAME "${APPLICATION_SHORTNAME}_${CFAPI_SHELL_EXTENSIONS_LIB_NAME}_WndClass" )
 
 set( CFAPI_SHELLEXT_APPID_REG "{E314A650-DCA4-416E-974E-18EA37C213EA}")
 set( CFAPI_SHELLEXT_APPID_DISPLAY_NAME "${APPLICATION_NAME} CfApi Shell Extensions" )

admin/win/msi/Nextcloud.wxs

@@ -60,18 +60,33 @@
 		    <FileSearch Id="LegacyUninstallFileName" Name="Uninstall.exe"/>
 	    </RegistrySearch>
     </Property>
+
+    <Property Id="IS_PREV_VERSION_SHELL_EXT_CLOSE_SUPPORTED">
+        <RegistrySearch Id="RegIsPrevVersionShellExtCloseSupported" Type="raw" Root="HKLM" Key="Software\$(var.AppVendor)\$(var.AppName)" Name="isShellExtCloseSupported" Win64="$(var.PlatformWin64)" />
+    </Property>
 
     <!-- Property to disable update checks -->
     <Property Id="SKIPAUTOUPDATE" Value="0" />
 
     <!-- Quit / restart application -->
     <util:RestartResource ProcessName="$(var.AppExe)" />
+
+    <Property Id="WNDCLASSNAMETOCLOSE" Value="$(var.CfApiShellextWndClassName)" />
+    <CustomAction Id="SetCfApiWindowClassName" Property="WNDCLASSNAMETOCLOSE" Value="$(var.CfApiShellextWndClassName)" />
+    <CustomAction Id="SetNCContextMenuWindowClassName" Property="WNDCLASSNAMETOCLOSE" Value="$(var.NCContextMenuShellextWndClassName)" />
+    <CustomAction Id="SetNCOverlaysWindowClassName" Property="WNDCLASSNAMETOCLOSE" Value="$(var.NCOverlaysShellextWndClassName)" />
 
-    <!-- Helper DLL Custom Actions -->
+      <!-- Helper DLL Custom Actions -->
     <SetProperty Id="ExecNsisUninstaller" Value="&quot;$(var.AppShortName)&quot; &quot;[NSIS_UNINSTALLEXE]&quot;" Before="ExecNsisUninstaller" Sequence="execute" />
     <SetProperty Id="RemoveNavigationPaneEntries" Value="&quot;$(var.AppName)&quot;" Before="RemoveNavigationPaneEntries" Sequence="execute" />
 
     <InstallExecuteSequence>
+      <Custom Action="SetCfApiWindowClassName" Before="InstallValidate"/>
+      <Custom Action="CloseCfApiShellExtension" After="SetCfApiWindowClassName" />
+      <Custom Action="SetNCContextMenuWindowClassName" After="CloseCfApiShellExtension"/>
+      <Custom Action="CloseNCContextMenuShellExtension" After="SetNCContextMenuWindowClassName" />
+      <Custom Action="SetNCOverlaysWindowClassName" After="CloseNCContextMenuShellExtension"/>
+      <Custom Action="CloseNCOverlaysShellExtension" After="SetNCOverlaysWindowClassName" />
         <!-- Install: Remove previous NSIS installation, if detected -->
         <Custom Action="ExecNsisUninstaller" Before="ProcessComponents">NSIS_UNINSTALLEXE AND NOT Installed</Custom>
 
@@ -80,9 +95,8 @@
 
 		<!-- Uninstall: Cleanup the Registry -->
 		<Custom Action="RegistryCleanupCustomAction" After="RemoveFiles">(NOT UPGRADINGPRODUCTCODE) AND (REMOVE="ALL")</Custom>
-
-        <!-- Schedule Reboot for the Shell Extensions (in silent installation mode only, or if SCHEDULE_REBOOT argument is set-->
-        <ScheduleReboot After="InstallFinalize">(SCHEDULE_REBOOT=1) OR NOT (UILevel=2)</ScheduleReboot>
+        <!-- Schedule Reboot for the Shell Extensions (only if SCHEDULE_REBOOT argument is set or if th version is less than 3.12.2 -->
+      <ScheduleReboot After="InstallFinalize">(SCHEDULE_REBOOT=1) OR ((NOT (UILevel=2)) AND (NOT IS_PREV_VERSION_SHELL_EXT_CLOSE_SUPPORTED))</ScheduleReboot>
     </InstallExecuteSequence>
 
     <!-- "Add or Remove" Programs Entries -->
@@ -184,6 +198,11 @@
                 <RegistryValue Type="string" Name="InstallerProductCode" Value="[ProductCode]" />
             </RegistryKey>
         </Component>
+      <Component Id="RegistryMiscInfo" Guid="*" Win64="$(var.PlatformWin64)">
+        <RegistryKey Root="HKLM" Key="Software\$(var.AppVendor)\$(var.AppName)">
+          <RegistryValue Type="integer" Name="isShellExtCloseSupported" Value="1" />
+        </RegistryKey>
+      </Component>
 
         <!-- Platform bitness-dependent settings -->
         <Component Id="RegistryDefaultSettings" Guid="*" Win64="$(var.PlatformWin64)">
@@ -213,6 +232,7 @@
         <ComponentGroupRef Id="ClientFiles" />
 
         <ComponentRef Id="RegistryVersionInfo" />
+        <ComponentRef Id="RegistryMiscInfo" />
         <ComponentRef Id="RegistryDefaultSettings" />
 		<ComponentRef Id="RegistryUriHandler" />
 

admin/win/msi/OEM.wxi.in

@@ -31,6 +31,10 @@
 
 	<?define AppCommandOpenUrlScheme = "@APPLICATION_URI_HANDLER_SCHEME@" ?>
 
+    <?define CfApiShellextWndClassName = "@CFAPI_SHELLEXT_WINDOW_CLASS_NAME@" ?>
+    <?define NCOverlaysShellextWndClassName = "@NCOVERLAYS_SHELLEXT_WINDOW_CLASS_NAME@" ?>
+    <?define NCContextMenuShellextWndClassName = "@NCCONTEXTMENU_SHELLEXT_WINDOW_CLASS_NAME@" ?>
+
     <!-- Custom license: To use it, also remove the "Skip the license page" stuff in the <UI> section
                          and uncomment <WixVariable Id="WixUILicenseRtf"...
     <?define AppLicenseRtf = "path\License.rtf" ?>

admin/win/tools/NCMsiHelper/CustomAction.cpp

@@ -1,4 +1,4 @@
 /*
  * Copyright (C) by Michael Schuster <[email protected]>
  *
  * This program is free software; you can redistribute it and/or modify
@@ -20,6 +20,8 @@
  */
 
 #include "NCMsiHelper.h"
+#include <MsiQuery.h>
+#include <vector>
 
 /**
  *  Sets up logging for MSIs and then calls the appropriate custom action with argc/argv parameters.
@@ -94,6 +96,71 @@
     return CustomActionArgcArgv(hInstall, DoRemoveNavigationPaneEntries, "RemoveNavigationPaneEntries");
 }
 
+UINT LogMsiInfoMessage(MSIHANDLE hInstall, const TCHAR *format, ...)
+{
+    TCHAR szFormatted[MAX_PATH];
+
+    va_list args;
+    va_start(args, format);
+    vswprintf(szFormatted, MAX_PATH, format, args);
+    va_end(args);
+
+    PMSIHANDLE hRecord = ::MsiCreateRecord(1);
+    ::MsiRecordSetString(hRecord, 0, szFormatted);
+
+    // we are always logging a message as info, as error will bring a popup that we don't want, just logs
+    return MsiProcessMessage(hInstall, INSTALLMESSAGE_INFO, hRecord);
+}
+
+UINT __stdcall CloseWindowByClassName(MSIHANDLE hInstall)
+{
+    const auto windowClassPropertyName = _T("WNDCLASSNAMETOCLOSE");
+    DWORD windowClassNameSize = 0;
+    if (MsiGetProperty(hInstall, windowClassPropertyName, _T(""), &windowClassNameSize) != ERROR_MORE_DATA) {
+        LogMsiInfoMessage(hInstall,
+                          _T("ERROR: Custom action CloseWindowByClassName. MsiGetProperty failed for windowClassPropertyName: %s"),
+                          windowClassPropertyName);
+        return ERROR_BAD_ARGUMENTS;
+    }
+
+    if (windowClassNameSize <= 0) {
+        LogMsiInfoMessage(hInstall, _T("ERROR: Custom action CloseWindowByClassName. classNameSize is <= 0!"));
+        return ERROR_BAD_ARGUMENTS;
+    }
+
+    ++windowClassNameSize;
+
+    std::vector<TCHAR> windowClassNameValue(windowClassNameSize, 0);
+    std::vector<char> vec;
+    const auto getPropertyRes = MsiGetProperty(hInstall, windowClassPropertyName, windowClassNameValue.data(), &windowClassNameSize);
+    if (getPropertyRes != ERROR_SUCCESS) {
+        LogMsiInfoMessage(hInstall, _T("ERROR: Custom action CloseWindowByClassName. MsiGetProperty failed for windowClassPropertyName: %s with code: %d"),
+            windowClassNameValue.data(),
+            getPropertyRes);
+        return getPropertyRes;
+    }
+
+    if (windowClassNameSize <= 0) {
+        LogMsiInfoMessage(hInstall, _T("ERROR: Custom action CloseWindowByClassName. Final classNameSize is <= 0!"));
+        return ERROR_BAD_ARGUMENTS;
+    }
+
+    LogMsiInfoMessage(hInstall, _T("Custom action CloseWindowByClassName is running for windowClassNameValue: %s"), windowClassNameValue.data());
+
+    const auto windowToCloseHandle = FindWindow(windowClassNameValue.data(), NULL);
+    if (windowToCloseHandle == NULL) {
+        LogMsiInfoMessage(hInstall, _T("WARNING: Custom action CloseWindowByClassName. windowToCloseHandle is NULL."));
+        // FindWindow will return NULL if the window is not currently running, so not an error
+        return ERROR_SUCCESS;
+    }
+
+    LogMsiInfoMessage(hInstall, _T("Custom action CloseWindowByClassName. Sending WM_CLOSE message to windowClassNameValue: %s"), windowClassNameValue.data());
+
+    SendMessage(windowToCloseHandle, WM_CLOSE, 0, 0);
+
+    return ERROR_SUCCESS;
+}
+
 /**
  * DllMain - Initialize and cleanup WiX custom action utils.
  */

admin/win/tools/NCMsiHelper/CustomAction.def

@@ -1,3 +1,4 @@
 EXPORTS
+CloseWindowByClassName
 ExecNsisUninstaller
 RemoveNavigationPaneEntries

admin/win/tools/NCMsiHelper/NCMsiHelper.wxs

@@ -39,5 +39,24 @@
                     Execute="deferred"
                     Impersonate="yes" />
 
+      <CustomAction Id="CloseCfApiShellExtension"
+                    Return="ignore"
+                    BinaryKey="NCMsiHelper"
+                    DllEntry="CloseWindowByClassName"
+                    Execute="immediate"
+                    Impersonate="yes" />
+      <CustomAction Id="CloseNCContextMenuShellExtension"
+                     Return="ignore"
+                     BinaryKey="NCMsiHelper"
+                     DllEntry="CloseWindowByClassName"
+                     Execute="immediate"
+                     Impersonate="yes" />
+      <CustomAction Id="CloseNCOverlaysShellExtension"
+                    Return="ignore"
+                    BinaryKey="NCMsiHelper"
+                    DllEntry="CloseWindowByClassName"
+                    Execute="immediate"
+                    Impersonate="yes" />
+
     </Fragment>
 </Wix>

config.h.in

@@ -48,6 +48,10 @@
 
 #cmakedefine BUILD_UPDATER "@BUILD_UPDATER@"
 
+#cmakedefine CFAPI_SHELLEXT_WINDOW_CLASS_NAME "@CFAPI_SHELLEXT_WINDOW_CLASS_NAME@"
+#cmakedefine NCCONTEXTMENU_SHELLEXT_WINDOW_CLASS_NAME "@NCCONTEXTMENU_SHELLEXT_WINDOW_CLASS_NAME@"
+#cmakedefine NCOVERLAYS_SHELLEXT_WINDOW_CLASS_NAME "@NCOVERLAYS_SHELLEXT_WINDOW_CLASS_NAME@"
+
 #cmakedefine CFAPI_SHELLEXT_APPID_REG "@CFAPI_SHELLEXT_APPID_REG@"
 #cmakedefine CFAPI_SHELLEXT_APPID_DISPLAY_NAME "@CFAPI_SHELLEXT_APPID_DISPLAY_NAME@"
 

shell_integration/windows/NCContextMenu/dllmain.cpp

@@ -12,7 +12,7 @@
 * details.
 */
 
 #include <windows.h>
 #include <Guiddef.h>
 #include "NCContextMenuRegHandler.h"
 #include "NCContextMenuFactory.h"
@@ -21,6 +21,11 @@
 HINSTANCE   g_hInst = nullptr;
 long        g_cDllRef = 0;
 
+HWND hHiddenWnd = nullptr;
+DWORD WINAPI MessageLoopThread(LPVOID lpParameter);
+LRESULT CALLBACK HiddenWndProc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam);
+void CreateHiddenWindowAndLaunchMessageLoop();
+
 BOOL APIENTRY DllMain(HMODULE hModule, DWORD dwReason, LPVOID lpReserved)
 {
     switch (dwReason)
@@ -30,6 +35,7 @@
         // path of the DLL to register the component.
         g_hInst = hModule;
         DisableThreadLibraryCalls(hModule);
+        CreateHiddenWindowAndLaunchMessageLoop();
         break;
     case DLL_THREAD_ATTACH:
     case DLL_THREAD_DETACH:
@@ -122,3 +128,63 @@
 
     return hr;
 }
+
+void CreateHiddenWindowAndLaunchMessageLoop()
+{
+    const WNDCLASSEX hiddenWindowClass{sizeof(WNDCLASSEX),
+                                       CS_CLASSDC,
+                                       HiddenWndProc,
+                                       0L,
+                                       0L,
+                                       GetModuleHandle(NULL),
+                                       NULL,
+                                       NULL,
+                                       NULL,
+                                       NULL,
+                                       NCCONTEXTMENU_SHELLEXT_WINDOW_CLASS_NAME,
+                                       NULL};
+
+    RegisterClassEx(&hiddenWindowClass);
+
+    hHiddenWnd = CreateWindow(hiddenWindowClass.lpszClassName,
+                              L"",
+                              WS_OVERLAPPEDWINDOW,
+                              CW_USEDEFAULT,
+                              CW_USEDEFAULT,
+                              CW_USEDEFAULT,
+                              CW_USEDEFAULT,
+                              NULL,
+                              NULL,
+                              hiddenWindowClass.hInstance,
+                              NULL);
+
+    ShowWindow(hHiddenWnd, SW_HIDE);
+    UpdateWindow(hHiddenWnd);
+
+    const auto hMessageLoopThread = CreateThread(NULL, 0, MessageLoopThread, NULL, 0, NULL);
+    if (hMessageLoopThread) {
+        CloseHandle(hMessageLoopThread);
+    }
+}
+
+DWORD WINAPI MessageLoopThread(LPVOID lpParameter)
+{
+    MSG msg;
+    while (GetMessage(&msg, NULL, 0, 0)) {
+        TranslateMessage(&msg);
+        DispatchMessage(&msg);
+    }
+    return 0;
+}
+
+LRESULT CALLBACK HiddenWndProc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam)
+{
+    switch (msg) {
+    case WM_CLOSE:
+        FreeLibrary(g_hInst);
+        break;
+    default:
+        return DefWindowProc(hwnd, msg, wParam, lParam);
+    }
+    return 0;
+}

shell_integration/windows/NCOverlays/DllMain.cpp

@@ -20,13 +20,19 @@ HINSTANCE instanceHandle = nullptr;
 
 long dllReferenceCount = 0;
 
+HWND hHiddenWnd = nullptr;
+DWORD WINAPI MessageLoopThread(LPVOID lpParameter);
+LRESULT CALLBACK HiddenWndProc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam);
+void CreateHiddenWindowAndLaunchMessageLoop();
+
 BOOL APIENTRY DllMain(HMODULE hModule, DWORD dwReason, LPVOID lpReserved)
 {
     switch (dwReason)
     {
         case DLL_PROCESS_ATTACH:
             instanceHandle = hModule;
             DisableThreadLibraryCalls(hModule);
+            CreateHiddenWindowAndLaunchMessageLoop();
             break;
         case DLL_THREAD_ATTACH:
         case DLL_THREAD_DETACH:
@@ -175,3 +181,63 @@ STDAPI DllUnregisterServer(void)
 
     return hResult;
 }
+
+void CreateHiddenWindowAndLaunchMessageLoop()
+{
+    const WNDCLASSEX hiddenWindowClass{sizeof(WNDCLASSEX),
+                                       CS_CLASSDC,
+                                       HiddenWndProc,
+                                       0L,
+                                       0L,
+                                       GetModuleHandle(NULL),
+                                       NULL,
+                                       NULL,
+                                       NULL,
+                                       NULL,
+                                       NCOVERLAYS_SHELLEXT_WINDOW_CLASS_NAME,
+                                       NULL};
+
+    RegisterClassEx(&hiddenWindowClass);
+
+    hHiddenWnd = CreateWindow(hiddenWindowClass.lpszClassName,
+                              L"",
+                              WS_OVERLAPPEDWINDOW,
+                              CW_USEDEFAULT,
+                              CW_USEDEFAULT,
+                              CW_USEDEFAULT,
+                              CW_USEDEFAULT,
+                              NULL,
+                              NULL,
+                              hiddenWindowClass.hInstance,
+                              NULL);
+
+    ShowWindow(hHiddenWnd, SW_HIDE);
+    UpdateWindow(hHiddenWnd);
+
+    const auto hMessageLoopThread = CreateThread(NULL, 0, MessageLoopThread, NULL, 0, NULL);
+    if (hMessageLoopThread) {
+        CloseHandle(hMessageLoopThread);
+    }
+}
+
+DWORD WINAPI MessageLoopThread(LPVOID lpParameter)
+{
+    MSG msg;
+    while (GetMessage(&msg, NULL, 0, 0)) {
+        TranslateMessage(&msg);
+        DispatchMessage(&msg);
+    }
+    return 0;
+}
+
+LRESULT CALLBACK HiddenWndProc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam)
+{
+    switch (msg) {
+    case WM_CLOSE:
+        FreeLibrary(instanceHandle);
+        break;
+    default:
+        return DefWindowProc(hwnd, msg, wParam, lParam);
+    }
+    return 0;
+}

shell_integration/windows/WinShellExtConstants.h.in

@@ -29,6 +29,9 @@
 #define OVERLAY_GUID_SYNC           L"@WIN_SHELLEXT_OVERLAY_GUID_SYNC@"
 #define OVERLAY_GUID_WARNING        L"@WIN_SHELLEXT_OVERLAY_GUID_WARNING@"
 
+#cmakedefine NCCONTEXTMENU_SHELLEXT_WINDOW_CLASS_NAME L"@NCCONTEXTMENU_SHELLEXT_WINDOW_CLASS_NAME@"
+#cmakedefine NCOVERLAYS_SHELLEXT_WINDOW_CLASS_NAME L"@NCOVERLAYS_SHELLEXT_WINDOW_CLASS_NAME@"
+
 //
 // Preceding spaces are intended, two spaces to put us ahead of the competition :/
 //