Changing windows close button hover colour

[kai9987kai]

I have been trying to change the windows close button hover colour. I have had dll issues and unexpected results such as transparent windows and crashes. I cant seem to get this idea to work any help would be appreciated my last attempt is below which currently makes windows invisible and unclickable. There have been issues with dwmapi.dll.

// ==WindhawkMod==
// @id              close-button-hover-color-changer
// @name            Close Button Hover Color Changer
// @description     Attempts to change the close button's hover color safely on Windows 11
// @version         2.0
// @author          AssistantModder
// @github          https://github.com/AssistantModder
// @include         *
// @compilerOptions -ldwmapi -luser32 -lgdi32
// ==/WindhawkMod==

// ==WindhawkModReadme==
/*
# Close Button Hover Color Changer

This mod attempts to safely customize the hover color of the close button across applications that use the Windows DWM API, specifically tailored for Windows 11.

## Features
- Attempts to change the hover color of the close button.
- Safe approach to avoid unintended side effects.
- Detailed logging for troubleshooting.

Note: This mod is experimental and may not work for all applications or Windows versions, especially modern UWP apps in Windows 11.
*/
// ==/WindhawkModReadme==

// ==WindhawkModSettings==
/*
- closeButtonHoverColor: "#FF6666"
  $name: Hover Color
  $description: Color when hovering over the close button (e.g., #FF0000 for red, #00FF00 for green).
- enableMod: true
  $name: Mod Enabled
  $description: Toggle the mod on or off.
- logLevel: 4
  $name: Logging Level
  $description: Set logging level - 0:None, 1:Error, 2:Warning, 3:Info, 4:Debug
*/
// ==/WindhawkModSettings==

#include <windows.h>
#include <dwmapi.h>
#include <stdio.h>

// Structure to hold the mod's settings
struct ModSettings {
    COLORREF hoverColor;
    BOOL enableMod;
    int logLevel;
} settings;

// Function prototype for the DWM API call
typedef HRESULT (WINAPI *DwmSetWindowAttributeFptr)(HWND, DWORD, LPCVOID, DWORD);

// Original function pointer
DwmSetWindowAttributeFptr DwmSetWindowAttribute_Original;

// Converts a hex string to COLORREF
COLORREF HexStringToCOLORREF(const wchar_t* hexColor) {
    unsigned int r, g, b;
    if (hexColor && wcslen(hexColor) == 7 && swscanf(hexColor, L"#%02x%02x%02x", &r, &g, &b) == 3) {
        return RGB(r, g, b);
    }
    Wh_Log(L"Error: Invalid hex color format '%s'. Using default red.", hexColor);
    return RGB(255, 0, 0); // Default to red on failure
}

// Custom logging function based on the log level
void CustomLog(int level, const wchar_t* format, ...) {
    if (level <= settings.logLevel) {
        va_list args;
        va_start(args, format);
        wchar_t buffer[256];
        vswprintf(buffer, sizeof(buffer) / sizeof(wchar_t), format, args);
        va_end(args);
        Wh_Log(L"%s", buffer);
    }
}

// Hook function for DwmSetWindowAttribute to change close button hover color
HRESULT WINAPI DwmSetWindowAttribute_Hook(HWND hwnd, DWORD dwAttribute, LPCVOID pvAttribute, DWORD cbAttribute) {
    if (settings.enableMod && pvAttribute && cbAttribute == sizeof(COLORREF)) {
        COLORREF* colorPtr = (COLORREF*)pvAttribute;
        COLORREF originalColor = *colorPtr;

        // Log only the close button hover color attribute
        if (dwAttribute == DWMWA_CAPTION_BUTTON_BOUNDS) { 
            CustomLog(4, L"DwmSetWindowAttribute: Attribute %d, Original Color: 0x%06X", 
                      dwAttribute, originalColor);
            *colorPtr = settings.hoverColor;
            CustomLog(4, L"DwmSetWindowAttribute: Attribute %d changed from 0x%06X to 0x%06X", 
                      dwAttribute, originalColor, settings.hoverColor);
        }
    }
    return DwmSetWindowAttribute_Original(hwnd, dwAttribute, pvAttribute, cbAttribute);
}

// Load settings from the Windhawk mod settings
void LoadSettings() {
    settings.hoverColor = HexStringToCOLORREF(Wh_GetStringSetting(L"closeButtonHoverColor"));
    settings.enableMod = Wh_GetIntSetting(L"enableMod") != 0;
    settings.logLevel = Wh_GetIntSetting(L"logLevel");
    CustomLog(3, L"Settings loaded: HoverColor=0x%06X, EnableMod=%d, LogLevel=%d", 
              settings.hoverColor, settings.enableMod, settings.logLevel);
}

// Initialization function for the mod
BOOL Wh_ModInit() {
    LoadSettings();

    HMODULE dwmapiModule = GetModuleHandle(L"dwmapi.dll");
    if (!dwmapiModule) {
        dwmapiModule = LoadLibrary(L"dwmapi.dll");
        if (!dwmapiModule) {
            CustomLog(1, L"Failed to load dwmapi.dll module");
            return FALSE;
        }
    }

    DwmSetWindowAttribute_Original = (DwmSetWindowAttributeFptr)GetProcAddress(dwmapiModule, "DwmSetWindowAttribute");
    if (!DwmSetWindowAttribute_Original) {
        CustomLog(1, L"Failed to get DwmSetWindowAttribute function address");
        return FALSE;
    }

    if (!Wh_SetFunctionHook((void*)DwmSetWindowAttribute_Original, (void*)DwmSetWindowAttribute_Hook, (void**)&DwmSetWindowAttribute_Original)) {
        CustomLog(1, L"Failed to set hook for DwmSetWindowAttribute");
        return FALSE;
    }

    CustomLog(3, L"Mod initialized successfully");
    return TRUE;
}

// Function called when the mod settings are changed
void Wh_ModSettingsChanged() {
    LoadSettings();
    CustomLog(3, L"Settings reloaded");
}

// Uninitialization function for the mod
void Wh_ModUninit() {
    CustomLog(3, L"Mod uninitialized");
}

[kai9987kai]

New attempt with a few changes but still does not do what is intended

// ==WindhawkMod==
// @id              close-button-hover-color-changer
// @name            Close Button Hover Color Changer
// @description     Changes the close button's hover color safely on Windows 11
// @version         2.1
// @author          AssistantModder
// @github          https://github.com/AssistantModder
// @include         *
// @compilerOptions -ldwmapi -luser32 -lgdi32
// ==/WindhawkMod==

// ==WindhawkModReadme==
/*
# Close Button Hover Color Changer

This mod changes the hover color of the close button across applications that use the Windows DWM API, specifically tailored for Windows 11.

## Features
- Changes the hover color of the close button.
- Safe approach to avoid unintended side effects.
- Detailed logging for troubleshooting.

Note: This mod is designed for Windows 11 and may not work on earlier versions of Windows.
*/
// ==/WindhawkModReadme==

// ==WindhawkModSettings==
/*
- closeButtonHoverColor: "#FF6666"
  $name: Hover Color
  $description: Color when hovering over the close button (e.g., #FF0000 for red, #00FF00 for green).
- enableMod: true
  $name: Mod Enabled
  $description: Toggle the mod on or off.
- logLevel: 4
  $name: Logging Level
  $description: Set logging level - 0:None, 1:Error, 2:Warning, 3:Info, 4:Debug
*/
// ==/WindhawkModSettings==

#include <windows.h>
#include <dwmapi.h>
#include <stdio.h>

// Structure to hold the mod's settings
struct ModSettings {
    COLORREF hoverColor;
    BOOL enableMod;
    int logLevel;
} settings;

// Function prototype for the DWM API call
typedef HRESULT (WINAPI *DwmSetWindowAttributeFptr)(HWND, DWORD, LPCVOID, DWORD);

// Original function pointer
DwmSetWindowAttributeFptr DwmSetWindowAttribute_Original;

// Converts a hex string to COLORREF
COLORREF HexStringToCOLORREF(const wchar_t* hexColor) {
    unsigned int r, g, b;
    if (hexColor && wcslen(hexColor) == 7 && swscanf(hexColor, L"#%02x%02x%02x", &r, &g, &b) == 3) {
        return RGB(r, g, b);
    }
    Wh_Log(L"Error: Invalid hex color format '%s'. Using default red.", hexColor);
    return RGB(255, 0, 0); // Default to red on failure
}

// Custom logging function based on the log level
void CustomLog(int level, const wchar_t* format, ...) {
    if (level <= settings.logLevel) {
        va_list args;
        va_start(args, format);
        wchar_t buffer[256];
        vswprintf(buffer, sizeof(buffer) / sizeof(wchar_t), format, args);
        va_end(args);
        Wh_Log(L"%s", buffer);
    }
}

// Hook function for DwmSetWindowAttribute to change close button hover color
HRESULT WINAPI DwmSetWindowAttribute_Hook(HWND hwnd, DWORD dwAttribute, LPCVOID pvAttribute, DWORD cbAttribute) {
    if (settings.enableMod && dwAttribute == DWMWA_CAPTION_COLOR && pvAttribute && cbAttribute == sizeof(COLORREF)) {
        COLORREF* colorPtr = (COLORREF*)pvAttribute;
        COLORREF originalColor = *colorPtr;

        CustomLog(4, L"DwmSetWindowAttribute: Attribute %d, Original Color: 0x%06X", 
                  dwAttribute, originalColor);
        *colorPtr = settings.hoverColor;
        CustomLog(4, L"DwmSetWindowAttribute: Attribute %d changed from 0x%06X to 0x%06X", 
                  dwAttribute, originalColor, settings.hoverColor);
        
        return DwmSetWindowAttribute_Original(hwnd, dwAttribute, colorPtr, cbAttribute);
    }
    
    return DwmSetWindowAttribute_Original(hwnd, dwAttribute, pvAttribute, cbAttribute);
}

// Load settings from the Windhawk mod settings
void LoadSettings() {
    settings.hoverColor = HexStringToCOLORREF(Wh_GetStringSetting(L"closeButtonHoverColor"));
    settings.enableMod = Wh_GetIntSetting(L"enableMod") != 0;
    settings.logLevel = Wh_GetIntSetting(L"logLevel");
    CustomLog(3, L"Settings loaded: HoverColor=0x%06X, EnableMod=%d, LogLevel=%d", 
              settings.hoverColor, settings.enableMod, settings.logLevel);
}

// Initialization function for the mod
BOOL Wh_ModInit() {
    LoadSettings();

    HMODULE dwmapiModule = GetModuleHandle(L"dwmapi.dll");
    if (!dwmapiModule) {
        dwmapiModule = LoadLibrary(L"dwmapi.dll");
        if (!dwmapiModule) {
            CustomLog(1, L"Failed to load dwmapi.dll module");
            return FALSE;
        }
    }

    DwmSetWindowAttribute_Original = (DwmSetWindowAttributeFptr)GetProcAddress(dwmapiModule, "DwmSetWindowAttribute");
    if (!DwmSetWindowAttribute_Original) {
        CustomLog(1, L"Failed to get DwmSetWindowAttribute function address");
        return FALSE;
    }

    if (!Wh_SetFunctionHook((void*)DwmSetWindowAttribute_Original, (void*)DwmSetWindowAttribute_Hook, (void**)&DwmSetWindowAttribute_Original)) {
        CustomLog(1, L"Failed to set hook for DwmSetWindowAttribute");
        return FALSE;
    }

    CustomLog(3, L"Mod initialized successfully");
    return TRUE;
}

// Function called when the mod settings are changed
void Wh_ModSettingsChanged() {
    LoadSettings();
    CustomLog(3, L"Settings reloaded");
}

// Uninitialization function for the mod
void Wh_ModUninit() {
    CustomLog(3, L"Mod uninitialized");
}

[silverbacknet]

That approach was never going to work anyway, because it would only change the entire title bar (the "caption") at once. You would have to disable standard painting and manually paint not only the buttons yourself, which is the purpose of DWMWA_CAPTION_BUTTON_BOUNDS, but the entire title bar.

You can get an idea of how complex this actually is here: https://github.com/grassator/win32-window-custom-titlebar/blob/main/main.c But, implementing all of that in Windhawk should actually work to give you the custom hover colors you want.