From 215ba27906afc38cc3f076a5999fa5ff970dddbb Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 29 Oct 2024 07:52:42 +0100 Subject: [PATCH 001/230] ubuntu fix --- install-early.sh | 4 +++- install-unstable.sh | 4 +++- install.sh | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/install-early.sh b/install-early.sh index b075eaee9..8797b70b4 100644 --- a/install-early.sh +++ b/install-early.sh @@ -2,6 +2,9 @@ linuxID=$(lsb_release -si) sandbox="" +if [ linuxID = "Ubuntu" ]; then + sandbox="--no-sandbox" +fi if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" @@ -52,7 +55,6 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail" - sandbox=" --no-sandbox" sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS diff --git a/install-unstable.sh b/install-unstable.sh index 913a175d0..040bb7cc8 100644 --- a/install-unstable.sh +++ b/install-unstable.sh @@ -2,6 +2,9 @@ linuxID=$(lsb_release -si) sandbox="" +if [ linuxID = "Ubuntu" ]; then + sandbox="--no-sandbox" +fi if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" @@ -52,7 +55,6 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail" - sandbox=" --no-sandbox" sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS diff --git a/install.sh b/install.sh index 04e52ef03..b3d1d9874 100644 --- a/install.sh +++ b/install.sh @@ -2,6 +2,9 @@ linuxID=$(lsb_release -si) sandbox="" +if [ linuxID = "Ubuntu" ]; then + sandbox="--no-sandbox" +fi if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" @@ -52,7 +55,7 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail" - sandbox=" --no-sandbox" + sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS From 00960e981534995356c88c4535dc18e06e6e1fd8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 11:16:27 +0100 Subject: [PATCH 002/230] make sure python is installed --- functions/helperFunctions.sh | 8 ++++++++ install-early.sh | 18 +++++++++++++----- install-unstable.sh | 18 +++++++++++++----- install.sh | 19 ++++++++++++++----- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/functions/helperFunctions.sh b/functions/helperFunctions.sh index 41ee8b8d6..334c05e89 100644 --- a/functions/helperFunctions.sh +++ b/functions/helperFunctions.sh @@ -1118,4 +1118,12 @@ function server_install(){ function startCompressor(){ konsole -e "/bin/bash $HOME/.config/EmuDeck/backend/tools/chdconv/chddeck.sh" +} + +function installPIP(){ + python -m pip --version &> /dev/null || python -m ensurepip --upgrade + python -m pip install --upgrade pip + grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc + grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc + PATH="$HOME/.local/bin:$PATH" } \ No newline at end of file diff --git a/install-early.sh b/install-early.sh index 8797b70b4..fd4bee355 100644 --- a/install-early.sh +++ b/install-early.sh @@ -5,6 +5,14 @@ sandbox="" if [ linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi +#Python PIP for steamOS +if [ $linuxID == "SteamOS" ]; then + python -m pip --version &> /dev/null || python -m ensurepip --upgrade + python -m pip install --upgrade pip + grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc + grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc + PATH="$HOME/.local/bin:$PATH" +fi if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" @@ -54,31 +62,31 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." - DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail" + DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS elif command -v pacman >/dev/null; then echo "Installing packages with pacman..." - ARCH_DEPS="steam jq zenity flatpak unzip bash fuse2 git rsync whiptail" + ARCH_DEPS="steam jq zenity flatpak unzip bash fuse2 git rsync whiptail python" sudo pacman --noconfirm -Syu sudo pacman --noconfirm -S $ARCH_DEPS elif command -v dnf >/dev/null; then echo "Installing packages with dnf..." - FEDORA_DEPS="jq zenity flatpak unzip bash fuse git rsync newt" + FEDORA_DEPS="jq zenity flatpak unzip bash fuse git rsync newt python" sudo dnf -y upgrade sudo dnf -y install $FEDORA_DEPS elif command -v zypper >/dev/null; then echo "Installing packages with zypper..." - SUSE_DEPS="steam jq zenity flatpak unzip bash libfuse2 git rsync whiptail" + SUSE_DEPS="steam jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" sudo zypper --non-interactive up sudo zypper --non-interactive install $SUSE_DEPS elif command -v xbps-install >/dev/null; then echo "Installing packages with xbps..." - VOID_DEPS="steam jq zenity flatpak unzip bash fuse git rsync whiptail" + VOID_DEPS="steam jq zenity flatpak unzip bash fuse git rsync whiptail python" sudo xbps-install -Syu sudo xbps-install -Sy $VOID_DEPS diff --git a/install-unstable.sh b/install-unstable.sh index 040bb7cc8..a0bb29f26 100644 --- a/install-unstable.sh +++ b/install-unstable.sh @@ -5,6 +5,14 @@ sandbox="" if [ linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi +#Python PIP for steamOS +if [ $linuxID == "SteamOS" ]; then + python -m pip --version &> /dev/null || python -m ensurepip --upgrade + python -m pip install --upgrade pip + grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc + grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc + PATH="$HOME/.local/bin:$PATH" +fi if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" @@ -54,31 +62,31 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." - DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail" + DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS elif command -v pacman >/dev/null; then echo "Installing packages with pacman..." - ARCH_DEPS="steam jq zenity flatpak unzip bash fuse2 git rsync whiptail" + ARCH_DEPS="steam jq zenity flatpak unzip bash fuse2 git rsync whiptail python" sudo pacman --noconfirm -Syu sudo pacman --noconfirm -S $ARCH_DEPS elif command -v dnf >/dev/null; then echo "Installing packages with dnf..." - FEDORA_DEPS="jq zenity flatpak unzip bash fuse git rsync newt" + FEDORA_DEPS="jq zenity flatpak unzip bash fuse git rsync newt python" sudo dnf -y upgrade sudo dnf -y install $FEDORA_DEPS elif command -v zypper >/dev/null; then echo "Installing packages with zypper..." - SUSE_DEPS="steam jq zenity flatpak unzip bash libfuse2 git rsync whiptail" + SUSE_DEPS="steam jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" sudo zypper --non-interactive up sudo zypper --non-interactive install $SUSE_DEPS elif command -v xbps-install >/dev/null; then echo "Installing packages with xbps..." - VOID_DEPS="steam jq zenity flatpak unzip bash fuse git rsync whiptail" + VOID_DEPS="steam jq zenity flatpak unzip bash fuse git rsync whiptail python" sudo xbps-install -Syu sudo xbps-install -Sy $VOID_DEPS diff --git a/install.sh b/install.sh index b3d1d9874..527261c0c 100644 --- a/install.sh +++ b/install.sh @@ -5,6 +5,15 @@ sandbox="" if [ linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi +#Python PIP for steamOS +if [ $linuxID == "SteamOS" ]; then + python -m pip --version &> /dev/null || python -m ensurepip --upgrade + python -m pip install --upgrade pip + grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc + grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc + PATH="$HOME/.local/bin:$PATH" +fi + if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" @@ -54,32 +63,32 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." - DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail" + DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS elif command -v pacman >/dev/null; then echo "Installing packages with pacman..." - ARCH_DEPS="steam jq zenity flatpak unzip bash fuse2 git rsync whiptail" + ARCH_DEPS="steam jq zenity flatpak unzip bash fuse2 git rsync whiptail python" sudo pacman --noconfirm -Syu sudo pacman --noconfirm -S $ARCH_DEPS elif command -v dnf >/dev/null; then echo "Installing packages with dnf..." - FEDORA_DEPS="jq zenity flatpak unzip bash fuse git rsync newt" + FEDORA_DEPS="jq zenity flatpak unzip bash fuse git rsync newt python" sudo dnf -y upgrade sudo dnf -y install $FEDORA_DEPS elif command -v zypper >/dev/null; then echo "Installing packages with zypper..." - SUSE_DEPS="steam jq zenity flatpak unzip bash libfuse2 git rsync whiptail" + SUSE_DEPS="steam jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" sudo zypper --non-interactive up sudo zypper --non-interactive install $SUSE_DEPS elif command -v xbps-install >/dev/null; then echo "Installing packages with xbps..." - VOID_DEPS="steam jq zenity flatpak unzip bash fuse git rsync whiptail" + VOID_DEPS="steam jq zenity flatpak unzip bash fuse git rsync whiptail python" sudo xbps-install -Syu sudo xbps-install -Sy $VOID_DEPS From 40308bbf669ca15b33d25590651fd0aaa60f564f Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 12:25:25 +0100 Subject: [PATCH 003/230] addGameListsArtwork --- functions/generateGameLists.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 6ffc9082e..35db4f333 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -159,4 +159,17 @@ saveImage(){ local dest_folder="$accountfolder/config/grid/emudeck" local dest_path="$dest_folder/$name.jpg" wget -q -O "$dest_path" "$url" -} \ No newline at end of file +} + +function addGameListsArtwork() { + # Parámetro de entrada + local file="$1" + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) + local appID=$(cat "$HOME/homebrew/settings/rom-library/scid.txt") + + + local origin="$accountFolder/config/grid/emudeck/$file.jpg" + local destination="$accountFolder/config/grid/${appID}p.jpg" + + ln -s "$origin" "$destination" +} From 4cfdcd3c1d7b8dda2f0bad4b537d1781dc096f0e Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 12:35:23 +0100 Subject: [PATCH 004/230] decky- --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 35db4f333..958f34757 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -165,7 +165,7 @@ function addGameListsArtwork() { # Parámetro de entrada local file="$1" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local appID=$(cat "$HOME/homebrew/settings/rom-library/scid.txt") + local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid.txt") local origin="$accountFolder/config/grid/emudeck/$file.jpg" From 24fc927eda7560871dfe40f7b4372b6e14cc0efc Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 12:53:09 +0100 Subject: [PATCH 005/230] $accountfolder --- functions/generateGameLists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 958f34757..6cef3c4af 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -168,8 +168,8 @@ function addGameListsArtwork() { local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid.txt") - local origin="$accountFolder/config/grid/emudeck/$file.jpg" - local destination="$accountFolder/config/grid/${appID}p.jpg" + local origin="$accountfolder/config/grid/emudeck/$file.jpg" + local destination="$accountfolder/config/grid/${appID}p.png" ln -s "$origin" "$destination" } From 23df1ff99e2c76af4653c2be6e2c47bf1219a185 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 12:55:23 +0100 Subject: [PATCH 006/230] more artwork --- functions/generateGameLists.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 6cef3c4af..fefa0de71 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -170,6 +170,10 @@ function addGameListsArtwork() { local origin="$accountfolder/config/grid/emudeck/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" + local destination_hero="$accountfolder/config/grid/${appID}_hero.png" + local destination_home="$accountfolder/config/grid/${appID}.png" ln -s "$origin" "$destination" + ln -s "$origin" "$destination_hero" + ln -s "$origin" "$destination_home" } From cc51fc8e920ff56efd6569aed4eec34c0e0042de Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 13:00:30 +0100 Subject: [PATCH 007/230] small fixes --- functions/generateGameLists.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index fefa0de71..8cba9706a 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -166,13 +166,13 @@ function addGameListsArtwork() { local file="$1" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid.txt") - - local origin="$accountfolder/config/grid/emudeck/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" local destination_home="$accountfolder/config/grid/${appID}.png" - + rm -rf "$destination" + rm -rf "$destination_hero" + rm -rf "$destination_home" ln -s "$origin" "$destination" ln -s "$origin" "$destination_hero" ln -s "$origin" "$destination_home" From ef9ca1a068c64b744223ca18215cf143cdd59567 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 13:38:44 +0100 Subject: [PATCH 008/230] chmod trick? --- functions/generateGameLists.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 8cba9706a..731657f04 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -163,6 +163,9 @@ saveImage(){ function addGameListsArtwork() { # Parámetro de entrada + chmod 777 $destination + chmod 777 $destination_hero + chmod 777 $destination_home local file="$1" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid.txt") @@ -176,4 +179,7 @@ function addGameListsArtwork() { ln -s "$origin" "$destination" ln -s "$origin" "$destination_hero" ln -s "$origin" "$destination_home" + chmod 444 $destination + chmod 444 $destination_hero + chmod 444 $destination_home } From f5c23feb12c04abd3cf998e7706b55751f177812 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 13:45:56 +0100 Subject: [PATCH 009/230] cp -rf --- functions/generateGameLists.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 731657f04..595f823fd 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -176,9 +176,9 @@ function addGameListsArtwork() { rm -rf "$destination" rm -rf "$destination_hero" rm -rf "$destination_home" - ln -s "$origin" "$destination" - ln -s "$origin" "$destination_hero" - ln -s "$origin" "$destination_home" + cp -rf "$origin" "$destination" + cp -rf "$origin" "$destination_hero" + cp -rf "$origin" "$destination_home" chmod 444 $destination chmod 444 $destination_hero chmod 444 $destination_home From c3fdf49091170f9e63a282eb3bd2bf2646ee3277 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 23:54:44 +0100 Subject: [PATCH 010/230] fixes dolphin control issues --- .../config/dolphin-emu/GCPadNew.ini | 8 ++++---- .../config/dolphin-emu/Hotkeys.ini | 2 +- .../config/dolphin-emu/Profiles/GCPad/GC_base.ini | 2 +- .../dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini | 2 +- .../dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini | 2 +- .../config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_NoController.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini | 2 +- .../Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini | 2 +- .../Profiles/Wiimote/Wii_classic_controller.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini | 2 +- .../Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini | 2 +- .../config/dolphin-emu/WiimoteNew.ini | 8 ++++---- 13 files changed, 19 insertions(+), 19 deletions(-) diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini index d6b564c05..359dae8ed 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini @@ -1,5 +1,5 @@ [GCPad1] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button W` Buttons/X = `Button E` @@ -30,7 +30,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad2] -Device = SDL/1/Steam Virtual Gamepad +Device =SteamDeck/1/Steam Deck Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` @@ -61,7 +61,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad3] -Device = SDL/2/Steam Virtual Gamepad +Device =SteamDeck/2/Steam Deck Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` @@ -92,7 +92,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad4] -Device = SDL/3/Steam Virtual Gamepad +Device =SteamDeck/3/Steam Deck Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini index 05d756ac6..8888aba79 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini @@ -1,5 +1,5 @@ [Hotkeys] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck General/Open = @(Ctrl+O) General/Toggle Pause = @(Back+`Button S`) General/Stop = @(Back+`Button W`) diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini index 31d3ecd26..d71d72af1 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button W` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini index 4a5961f00..56ca3becd 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button E` Buttons/X = `Button W` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini index 239090c22..1e3cfc7cf 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini index 4a5961f00..56ca3becd 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button E` Buttons/X = `Button W` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini index 98e6e81b3..f8e6e4d1e 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini @@ -1,2 +1,2 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini index 70e467723..1cc9ee83c 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini index 482b493ea..33bf99e3d 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini index 35b294e84..21ccf2107 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini index ae016fdf0..0b8e986d7 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini index 91725be6c..b2605c198 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini index 2ea37cb3d..d0e153cae 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini @@ -1,5 +1,5 @@ [Wiimote1] -Device = SDL/0/Steam Virtual Gamepad +Device =SteamDeck/0/Steam Deck Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -92,7 +92,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote2] -Device = SDL/1/Steam Virtual Gamepad +Device =SteamDeck/1/Steam Deck Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -185,7 +185,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote3] -Device = SDL/2/Steam Virtual Gamepad +Device =SteamDeck/2/Steam Deck Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -278,7 +278,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote4] -Device = SDL/3/Steam Virtual Gamepad +Device =SteamDeck/3/Steam Deck Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` From d925212b183959cb8ad917bdd045e088d8ec4717 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 23:56:49 +0100 Subject: [PATCH 011/230] Revert "fixes dolphin control issues" This reverts commit c3fdf49091170f9e63a282eb3bd2bf2646ee3277. --- .../config/dolphin-emu/GCPadNew.ini | 8 ++++---- .../config/dolphin-emu/Hotkeys.ini | 2 +- .../config/dolphin-emu/Profiles/GCPad/GC_base.ini | 2 +- .../dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini | 2 +- .../dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini | 2 +- .../config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_NoController.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini | 2 +- .../Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini | 2 +- .../Profiles/Wiimote/Wii_classic_controller.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini | 2 +- .../Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini | 2 +- .../config/dolphin-emu/WiimoteNew.ini | 8 ++++---- 13 files changed, 19 insertions(+), 19 deletions(-) diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini index 359dae8ed..d6b564c05 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini @@ -1,5 +1,5 @@ [GCPad1] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button W` Buttons/X = `Button E` @@ -30,7 +30,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad2] -Device =SteamDeck/1/Steam Deck +Device = SDL/1/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` @@ -61,7 +61,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad3] -Device =SteamDeck/2/Steam Deck +Device = SDL/2/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` @@ -92,7 +92,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad4] -Device =SteamDeck/3/Steam Deck +Device = SDL/3/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini index 8888aba79..05d756ac6 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini @@ -1,5 +1,5 @@ [Hotkeys] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad General/Open = @(Ctrl+O) General/Toggle Pause = @(Back+`Button S`) General/Stop = @(Back+`Button W`) diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini index d71d72af1..31d3ecd26 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button W` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini index 56ca3becd..4a5961f00 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button E` Buttons/X = `Button W` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini index 1e3cfc7cf..239090c22 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini index 56ca3becd..4a5961f00 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button E` Buttons/X = `Button W` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini index f8e6e4d1e..98e6e81b3 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini @@ -1,2 +1,2 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini index 1cc9ee83c..70e467723 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini index 33bf99e3d..482b493ea 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini index 21ccf2107..35b294e84 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini index 0b8e986d7..ae016fdf0 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini index b2605c198..91725be6c 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini @@ -1,5 +1,5 @@ [Profile] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini index d0e153cae..2ea37cb3d 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini @@ -1,5 +1,5 @@ [Wiimote1] -Device =SteamDeck/0/Steam Deck +Device = SDL/0/Steam Virtual Gamepad Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -92,7 +92,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote2] -Device =SteamDeck/1/Steam Deck +Device = SDL/1/Steam Virtual Gamepad Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -185,7 +185,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote3] -Device =SteamDeck/2/Steam Deck +Device = SDL/2/Steam Virtual Gamepad Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -278,7 +278,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote4] -Device =SteamDeck/3/Steam Deck +Device = SDL/3/Steam Virtual Gamepad Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` From 19e976b012495008ff364005cb701d14e63422f4 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 1 Nov 2024 23:57:32 +0100 Subject: [PATCH 012/230] dolphin for real --- .../config/dolphin-emu/GCPadNew.ini | 8 ++++---- .../config/dolphin-emu/Hotkeys.ini | 2 +- .../config/dolphin-emu/Profiles/GCPad/GC_base.ini | 2 +- .../dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini | 2 +- .../dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini | 2 +- .../config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_NoController.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini | 2 +- .../Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini | 2 +- .../Profiles/Wiimote/Wii_classic_controller.ini | 2 +- .../dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini | 2 +- .../Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini | 2 +- .../config/dolphin-emu/WiimoteNew.ini | 8 ++++---- 13 files changed, 19 insertions(+), 19 deletions(-) diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini index d6b564c05..058cd3556 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/GCPadNew.ini @@ -1,5 +1,5 @@ [GCPad1] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button W` Buttons/X = `Button E` @@ -30,7 +30,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad2] -Device = SDL/1/Steam Virtual Gamepad +Device = SDL/1/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` @@ -61,7 +61,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad3] -Device = SDL/2/Steam Virtual Gamepad +Device = SDL/2/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` @@ -92,7 +92,7 @@ D-Pad/Left = `Pad W` D-Pad/Right = `Pad E` Rumble/Motor = Strong [GCPad4] -Device = SDL/3/Steam Virtual Gamepad +Device = SDL/3/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini index 05d756ac6..d9ce2992b 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Hotkeys.ini @@ -1,5 +1,5 @@ [Hotkeys] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller General/Open = @(Ctrl+O) General/Toggle Pause = @(Back+`Button S`) General/Stop = @(Back+`Button W`) diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini index 31d3ecd26..2a3020541 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_base.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button W` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini index 4a5961f00..f407749c2 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_nintendo_layout.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button E` Buttons/X = `Button W` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini index 239090c22..ddc8b77b4 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_reverse_camera.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button N` Buttons/X = `Button E` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini index 4a5961f00..f407749c2 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/GCPad/GC_xbox_layout.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button E` Buttons/X = `Button W` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini index 98e6e81b3..23e885710 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_NoController.ini @@ -1,2 +1,2 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini index 70e467723..c34d31c4f 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini index 482b493ea..6c2b09d91 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_base_nunchuck_with_touchpad.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini index 35b294e84..134339f89 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_classic_controller.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini index ae016fdf0..45196df6f 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini index 91725be6c..04f0b3b6e 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/Profiles/Wiimote/Wii_no_attachment_with_touchpad.ini @@ -1,5 +1,5 @@ [Profile] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` diff --git a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini index 2ea37cb3d..43a6ecf56 100644 --- a/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini +++ b/configs/org.DolphinEmu.dolphin-emu/config/dolphin-emu/WiimoteNew.ini @@ -1,5 +1,5 @@ [Wiimote1] -Device = SDL/0/Steam Virtual Gamepad +Device = SDL/0/Steam Deck Controller Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -92,7 +92,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote2] -Device = SDL/1/Steam Virtual Gamepad +Device = SDL/1/Steam Deck Controller Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -185,7 +185,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote3] -Device = SDL/2/Steam Virtual Gamepad +Device = SDL/2/Steam Deck Controller Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` @@ -278,7 +278,7 @@ Drawsome/Stylus/Modifier/Range = 50. Rumble/Motor = Strong Options/Upright Wiimote = `Trigger L` [Wiimote4] -Device = SDL/3/Steam Virtual Gamepad +Device = SDL/3/Steam Deck Controller Buttons/A = `Button S`|`Thumb R`|`XInput2/0/Virtual core pointer:Click 1` Buttons/B = `Button E` Buttons/1 = `Button N` From 6b6f72dc018fa22a5d4d7eb4904ba72e0fe14117 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 2 Nov 2024 00:01:57 +0100 Subject: [PATCH 013/230] update versions --- versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.json b/versions.json index a571562f9..47b2675ea 100644 --- a/versions.json +++ b/versions.json @@ -1,6 +1,6 @@ { "ra": { "id": "ra", "code": "RetroArch", "version": 2 }, - "dolphin": { "id": "dolphin", "code": "Dolphin", "version": 3 }, + "dolphin": { "id": "dolphin", "code": "Dolphin", "version": 4 }, "primehack": { "id": "primehack", "code": "Primehack", "version": 0 }, "ppsspp": { "id": "ppsspp", "code": "PPSSPP", "version": 0 }, "duckstation": { "id": "duckstation", "code": "Duckstation", "version": 0 }, From 5b463a583bc51839f93a04963649e4b44d13188c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 2 Nov 2024 10:36:46 +0100 Subject: [PATCH 014/230] various scd --- functions/generateGameLists.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 595f823fd..5a012ca23 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -162,13 +162,10 @@ saveImage(){ } function addGameListsArtwork() { - # Parámetro de entrada - chmod 777 $destination - chmod 777 $destination_hero - chmod 777 $destination_home local file="$1" + local number="$2" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid.txt") + local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid_$number.txt") local origin="$accountfolder/config/grid/emudeck/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" @@ -179,7 +176,4 @@ function addGameListsArtwork() { cp -rf "$origin" "$destination" cp -rf "$origin" "$destination_hero" cp -rf "$origin" "$destination_home" - chmod 444 $destination - chmod 444 $destination_hero - chmod 444 $destination_home } From 8c5b946f4df4088154f49aced781f8e1d75c3fd0 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 2 Nov 2024 12:33:03 +0100 Subject: [PATCH 015/230] Revert "various scd" This reverts commit 5b463a583bc51839f93a04963649e4b44d13188c. --- functions/generateGameLists.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 5a012ca23..595f823fd 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -162,10 +162,13 @@ saveImage(){ } function addGameListsArtwork() { + # Parámetro de entrada + chmod 777 $destination + chmod 777 $destination_hero + chmod 777 $destination_home local file="$1" - local number="$2" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid_$number.txt") + local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid.txt") local origin="$accountfolder/config/grid/emudeck/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" @@ -176,4 +179,7 @@ function addGameListsArtwork() { cp -rf "$origin" "$destination" cp -rf "$origin" "$destination_hero" cp -rf "$origin" "$destination_home" + chmod 444 $destination + chmod 444 $destination_hero + chmod 444 $destination_home } From c9d61ffa0aebc9ecbd44163a4162e559f088795d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 2 Nov 2024 13:01:26 +0100 Subject: [PATCH 016/230] number cid --- functions/generateGameLists.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 595f823fd..5a012ca23 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -162,13 +162,10 @@ saveImage(){ } function addGameListsArtwork() { - # Parámetro de entrada - chmod 777 $destination - chmod 777 $destination_hero - chmod 777 $destination_home local file="$1" + local number="$2" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid.txt") + local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid_$number.txt") local origin="$accountfolder/config/grid/emudeck/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" @@ -179,7 +176,4 @@ function addGameListsArtwork() { cp -rf "$origin" "$destination" cp -rf "$origin" "$destination_hero" cp -rf "$origin" "$destination_home" - chmod 444 $destination - chmod 444 $destination_hero - chmod 444 $destination_home } From 700f2e723e602bfe4e600d1081659d6cb757ea28 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 2 Nov 2024 22:54:39 +0100 Subject: [PATCH 017/230] appID --- functions/generateGameLists.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 5a012ca23..07166983f 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -163,9 +163,8 @@ saveImage(){ function addGameListsArtwork() { local file="$1" - local number="$2" + local appID="$2" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local appID=$(cat "$HOME/homebrew/settings/decky-rom-library/scid_$number.txt") local origin="$accountfolder/config/grid/emudeck/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" From de2eccd2c56fac489236b77cf0bc6c117333f1e6 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 17:15:38 +0100 Subject: [PATCH 018/230] clean empty files --- functions/generateGameLists.sh | 9 ++++++++- tools/fuzzy_search_rom.py | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 07166983f..0a9864952 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -32,6 +32,13 @@ generateGameListsJson() { } +generateGameLists_importESDE() { + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) + local dest_folder="$accountfolder/config/grid/emudeck/" + + python $HOME/.config/EmuDeck/backend/tools/retroLibrary/import_media.py "$romsPath" "$dest_folder" +} + generateGameLists_artwork() { mkdir -p "$HOME/emudeck/cache/" local number_log=$1 @@ -69,7 +76,7 @@ generateGameLists_artwork() { if ! ls $file_to_check 1> /dev/null 2>&1 && [ -z "${processed_games[$game]}" ]; then echo "GAME:" "$game" >> "$logfilename" - fuzzygame=$(python $HOME/.config/EmuDeck/backend/tools/fuzzy_search_rom.py "$game") + fuzzygame=$(python $HOME/.config/EmuDeck/backend/tools/fuzzy_search_rom.py "$game" "$dest_folder") fuzzygame="${fuzzygame// /_}" fuzzygame="${fuzzygame//:/}" fuzzygame="${fuzzygame//./}" diff --git a/tools/fuzzy_search_rom.py b/tools/fuzzy_search_rom.py index 37c48a130..b00510b38 100644 --- a/tools/fuzzy_search_rom.py +++ b/tools/fuzzy_search_rom.py @@ -30,6 +30,7 @@ def is_file_older_than(file_path, days): # Título para buscar search_title = sys.argv[1] +images_path = sys.argv[2] #print(f"{search_title}") #sys.exit() @@ -50,6 +51,13 @@ def is_file_older_than(file_path, days): print(f"Ocurrió un error al ejecutar el comando: {e}") else: + for root, _, files in os.walk(images_path): + for file in files: + file_path = os.path.join(root, file) + # Verifica si el archivo tiene un tamaño de 0 bytes + if os.path.getsize(file_path) == 0: + os.remove(file_path) + # Cargar el JSON desde el disco duro with open(json_file_path, "r") as json_file: data = json_file.read() From 6519cda5c0f9b83d0ebc73b9f4bb6fec9bf9aa62 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 18:16:05 +0100 Subject: [PATCH 019/230] system_dir --- tools/generate_game_lists.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/generate_game_lists.py b/tools/generate_game_lists.py index 0630037fe..bc56f52f1 100644 --- a/tools/generate_game_lists.py +++ b/tools/generate_game_lists.py @@ -69,7 +69,8 @@ def collect_game_data(system_dir, extensions): "name": clean_name, "filename": file_path, "file": name_cleaned_pegasus, - "img": game_img + "img": game_img, + "platform": system_dir } game_data.append(game_info) game_data_sorted = sorted(game_data, key=lambda x: x['name']) From 2ed17e1875380541c1b42f4cd8a64ef75bddda98 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 18:20:16 +0100 Subject: [PATCH 020/230] platform --- tools/generate_game_lists.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/generate_game_lists.py b/tools/generate_game_lists.py index bc56f52f1..76c172e12 100644 --- a/tools/generate_game_lists.py +++ b/tools/generate_game_lists.py @@ -47,7 +47,7 @@ def collect_game_data(system_dir, extensions): if name == "ps3": name = parts[-1] - + platform = os.path.basename(system_dir) name_cleaned = re.sub(r'\(.*?\)', '', name) name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) @@ -70,7 +70,7 @@ def collect_game_data(system_dir, extensions): "filename": file_path, "file": name_cleaned_pegasus, "img": game_img, - "platform": system_dir + "platform": platform } game_data.append(game_info) game_data_sorted = sorted(game_data, key=lambda x: x['name']) From cb9c30b55515324882b09b5d5e88c27f69f5306d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 21:32:37 +0100 Subject: [PATCH 021/230] avoid zero k downloads --- functions/generateGameLists.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 0a9864952..fafb3ee28 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -160,12 +160,14 @@ generateGameLists_artwork() { } saveImage(){ - local url=$1 + local ""=$1 local name=$2 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/emudeck" local dest_path="$dest_folder/$name.jpg" - wget -q -O "$dest_path" "$url" + if [ $url != "" ]; then + wget -q -O "$dest_path" "$url" + fi } function addGameListsArtwork() { From 3665d939696bcedaa3245d6d068ccb31f2022be0 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 21:33:03 +0100 Subject: [PATCH 022/230] python fix --- tools/fuzzy_search_rom.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/fuzzy_search_rom.py b/tools/fuzzy_search_rom.py index b00510b38..70c7f329b 100644 --- a/tools/fuzzy_search_rom.py +++ b/tools/fuzzy_search_rom.py @@ -52,11 +52,11 @@ def is_file_older_than(file_path, days): else: for root, _, files in os.walk(images_path): - for file in files: - file_path = os.path.join(root, file) - # Verifica si el archivo tiene un tamaño de 0 bytes - if os.path.getsize(file_path) == 0: - os.remove(file_path) + for file in files: + file_path = os.path.join(root, file) + # Verifica si el archivo tiene un tamaño de 0 bytes + if os.path.getsize(file_path) == 0: + os.remove(file_path) # Cargar el JSON desde el disco duro with open(json_file_path, "r") as json_file: From cb35e570f09835af1307db4e29279a196b398cd3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 21:38:17 +0100 Subject: [PATCH 023/230] == --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index fafb3ee28..bc3209a9b 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -110,7 +110,7 @@ generateGameLists_artwork() { filename=$(basename "$game_img_url") dest_path="$dest_folder$game.jpg" - if [ "$game_img_url" = "null" ]; then + if [ "$game_img_url" == "null" ]; then echo -e " - No picture" >> "$logfilename" else echo "Added to the list (NO FUZZY): $game_img_url" >> "$logfilename" From 826197e899d68d2782d3bdd37acb39d595beb424 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 21:39:07 +0100 Subject: [PATCH 024/230] url --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index bc3209a9b..c77b29de3 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -160,7 +160,7 @@ generateGameLists_artwork() { } saveImage(){ - local ""=$1 + local url=$1 local name=$2 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/emudeck" From cfc20aa988856842d7e00a9e804f69e7177c04c8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 21:44:24 +0100 Subject: [PATCH 025/230] unary? --- functions/generateGameLists.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index c77b29de3..b7340eced 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -47,7 +47,7 @@ generateGameLists_artwork() { local json_file="$HOME/emudeck/cache/roms_games.json" local json=$(cat "$json_file") - if [ $number_log = 1 ]; then + if [ "$number_log" = 1 ]; then local platforms=$(echo "$json" | jq -r '.[].id') else local platforms=$(echo "$json" | jq -r '.[].id' | shuf) @@ -110,13 +110,13 @@ generateGameLists_artwork() { filename=$(basename "$game_img_url") dest_path="$dest_folder$game.jpg" - if [ "$game_img_url" == "null" ]; then - echo -e " - No picture" >> "$logfilename" - else + if [ "$game_img_url" != "null" ]; then echo "Added to the list (NO FUZZY): $game_img_url" >> "$logfilename" download_array+=("$game_img_url") download_dest_paths+=("$dest_path") processed_games[$game]=1 + else + echo -e " - No picture" >> "$logfilename" fi fi fi @@ -165,7 +165,7 @@ saveImage(){ local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/emudeck" local dest_path="$dest_folder/$name.jpg" - if [ $url != "" ]; then + if [ -n "$url" ] && [ -n "$name" ]; then wget -q -O "$dest_path" "$url" fi } From f5d2c85b92ebeba5335855a5d7e37e336a1122ef Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 7 Nov 2024 21:56:55 +0100 Subject: [PATCH 026/230] super quick comparison --- tools/fuzzy_search_rom.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/fuzzy_search_rom.py b/tools/fuzzy_search_rom.py index 70c7f329b..444f79509 100644 --- a/tools/fuzzy_search_rom.py +++ b/tools/fuzzy_search_rom.py @@ -9,18 +9,28 @@ def similar(a, b): return SequenceMatcher(None, a, b).ratio() +def simple_similarity(a, b): + # Calcula la longitud de la coincidencia de prefijo más larga + match_length = len([i for i in range(min(len(a), len(b))) if a[i] == b[i]]) + # Normaliza en función de la longitud del título de búsqueda + return match_length / max(len(a), len(b)) + def find_best_match(search_title, games): best_match = None highest_similarity = 0 for game in games: - similarity = similar(search_title, game) # Asegúrate de que estás comparando con el nombre del juego + # Calcula la similitud basada en el prefijo + similarity = simple_similarity(search_title, game) if similarity > highest_similarity: highest_similarity = similarity - best_match = game # Devolver solo el nombre del juego o el objeto completo si prefieres + best_match = game return best_match + + + def is_file_older_than(file_path, days): file_time = datetime.fromtimestamp(os.path.getmtime(file_path)) return datetime.now() - file_time > timedelta(days=days) From fd9b9f631f39d1e8bedf3db3ab19d4443060fa14 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 8 Nov 2024 09:38:27 +0100 Subject: [PATCH 027/230] generateGameLists_getPercentage --- functions/generateGameLists.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index b7340eced..37e5a8979 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -185,3 +185,13 @@ function addGameListsArtwork() { cp -rf "$origin" "$destination_hero" cp -rf "$origin" "$destination_home" } + +generateGameLists_getPercentage() { + local json_file="$HOME/emudeck/cache/roms_games.json" + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) + local destination="$accountfolder/config/grid/emudeck" + local jpg_count=$(find "$destination" -type f -name "*.jpg" | wc -l) + local games=$(jq '[.[].games[]] | length' "$json_file") + local percentage=$(( 100 * jpg_count / games )) + echo $percentage +} \ No newline at end of file From d24ddc952635110cfd982d7898b5a1bdab4cba5b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 8 Nov 2024 09:45:37 +0100 Subject: [PATCH 028/230] wget --- functions/generateGameLists.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 37e5a8979..6cc8e48f9 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -165,9 +165,7 @@ saveImage(){ local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/emudeck" local dest_path="$dest_folder/$name.jpg" - if [ -n "$url" ] && [ -n "$name" ]; then - wget -q -O "$dest_path" "$url" - fi + wget -q -O "$dest_path" "$url" } function addGameListsArtwork() { From 2b423ed68abab3a0f2e25bff5435d1db14634866 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 8 Nov 2024 15:33:23 +0100 Subject: [PATCH 029/230] % --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 6cc8e48f9..8bd9fa8d4 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -191,5 +191,5 @@ generateGameLists_getPercentage() { local jpg_count=$(find "$destination" -type f -name "*.jpg" | wc -l) local games=$(jq '[.[].games[]] | length' "$json_file") local percentage=$(( 100 * jpg_count / games )) - echo $percentage + echo "$jpg_count / $games ($percentage)" } \ No newline at end of file From 34aad96b2f3a7e1ff578ce0b843ae7c427cb67ac Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 8 Nov 2024 15:33:23 +0100 Subject: [PATCH 030/230] % --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 8bd9fa8d4..19a3ea562 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -191,5 +191,5 @@ generateGameLists_getPercentage() { local jpg_count=$(find "$destination" -type f -name "*.jpg" | wc -l) local games=$(jq '[.[].games[]] | length' "$json_file") local percentage=$(( 100 * jpg_count / games )) - echo "$jpg_count / $games ($percentage)" + echo "$jpg_count / $games ($percentage) %" } \ No newline at end of file From ff07116f7f6b2d1e6bf3022b2c2a45b4cc27f206 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 8 Nov 2024 20:47:07 +0100 Subject: [PATCH 031/230] % --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 19a3ea562..31592b80d 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -191,5 +191,5 @@ generateGameLists_getPercentage() { local jpg_count=$(find "$destination" -type f -name "*.jpg" | wc -l) local games=$(jq '[.[].games[]] | length' "$json_file") local percentage=$(( 100 * jpg_count / games )) - echo "$jpg_count / $games ($percentage) %" + echo "$jpg_count / $games ($percentage%)" } \ No newline at end of file From 9e77fc745d1625937362c488d9f8c677cf4015c5 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 11 Nov 2024 08:31:47 +0100 Subject: [PATCH 032/230] temp --- functions/generateGameLists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 31592b80d..458b8ddc8 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -87,7 +87,7 @@ generateGameLists_artwork() { #game_name=$(echo "$response" | jq -r '.name') #game_img_url=$(echo "$response" | jq -r '.img') - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg.php?name=$fuzzygame" + wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg2.php?name=$fuzzygame" game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") @@ -103,7 +103,7 @@ generateGameLists_artwork() { #response=$(curl -s -G "https://bot.emudeck.com/steamdbimg.php?name=$game") #game_name=$(echo "$response" | jq -r '.name') #game_img_url=$(echo "$response" | jq -r '.img') - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg.php?name=$game" + wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg2.php?name=$game" game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") From 08e2dc03f9be027f64314d8c7b8b9e6f9932c98a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 11 Nov 2024 15:17:13 +0100 Subject: [PATCH 033/230] Revert "temp" This reverts commit 9e77fc745d1625937362c488d9f8c677cf4015c5. --- functions/generateGameLists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 458b8ddc8..31592b80d 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -87,7 +87,7 @@ generateGameLists_artwork() { #game_name=$(echo "$response" | jq -r '.name') #game_img_url=$(echo "$response" | jq -r '.img') - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg2.php?name=$fuzzygame" + wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg.php?name=$fuzzygame" game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") @@ -103,7 +103,7 @@ generateGameLists_artwork() { #response=$(curl -s -G "https://bot.emudeck.com/steamdbimg.php?name=$game") #game_name=$(echo "$response" | jq -r '.name') #game_img_url=$(echo "$response" | jq -r '.img') - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg2.php?name=$game" + wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg.php?name=$game" game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") From 93034bd033e05324854b5368207ba325d7b19619 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 13:15:35 +0100 Subject: [PATCH 034/230] PID file --- functions/generateGameLists.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 31592b80d..274a5e9aa 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -41,12 +41,37 @@ generateGameLists_importESDE() { generateGameLists_artwork() { mkdir -p "$HOME/emudeck/cache/" + local number_log=$1 local current_time=$(date +"%H_%M_%S") local logfilename="$HOME/emudeck/logs/library_${number_log}.log" local json_file="$HOME/emudeck/cache/roms_games.json" local json=$(cat "$json_file") + local PIDFILE="$HOME/.config/EmuDeck/retrolibrary_$number_log.pid" + + if [ -f "$PIDFILE" ]; then + PID=$(cat "$PIDFILE") + ps -p "$PID" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "Process already running" + exit 1 + else + ## Process not found assume not running + echo $$ > "$PIDFILE" + if [ $? -ne 0 ]; then + echo "Could not create PID file" + exit 1 + fi + fi + else + echo $$ > "$PIDFILE" + if [ $? -ne 0 ]; then + echo "Could not create PID file" + exit 1 + fi + fi + if [ "$number_log" = 1 ]; then local platforms=$(echo "$json" | jq -r '.[].id') else @@ -157,6 +182,7 @@ generateGameLists_artwork() { # Save the updated JSON back to the file #echo "$json" > "$json_file" + rm -rf "$PIDFILE" } saveImage(){ From 4bff2be08e6050e8b31c546e220809b47bb32368 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 13:23:53 +0100 Subject: [PATCH 035/230] PID --- functions/generateGameLists.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 274a5e9aa..52d3078ba 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -51,19 +51,7 @@ generateGameLists_artwork() { local PIDFILE="$HOME/.config/EmuDeck/retrolibrary_$number_log.pid" if [ -f "$PIDFILE" ]; then - PID=$(cat "$PIDFILE") - ps -p "$PID" > /dev/null 2>&1 - if [ $? -eq 0 ]; then - echo "Process already running" - exit 1 - else - ## Process not found assume not running - echo $$ > "$PIDFILE" - if [ $? -ne 0 ]; then - echo "Could not create PID file" - exit 1 - fi - fi + exit 1 else echo $$ > "$PIDFILE" if [ $? -ne 0 ]; then From 7c3ac4b1d8dd3214ea93f39e85be946fb3925382 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 17:56:28 +0100 Subject: [PATCH 036/230] forced citra --- functions/EmuScripts/emuDeckCitra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckCitra.sh b/functions/EmuScripts/emuDeckCitra.sh index a8f03c3c6..0d10d0306 100755 --- a/functions/EmuScripts/emuDeckCitra.sh +++ b/functions/EmuScripts/emuDeckCitra.sh @@ -12,7 +12,7 @@ Citra_texturesPath="$HOME/.config/citra-emu/load/textures" Citra_install(){ echo "Begin $Citra_emuName Install" local showProgress="$1" - if installEmuAI "$Citra_emuName" "" "$(getReleaseURLGH "PabloMK7/citra" "tar.gz" "" "appimage")" "citra" "tar.gz" "emulator" "$showProgress"; then #citra-qt.AppImage + if installEmuAI "$Citra_emuName" "" "https://github.com/PabloMK7/citra/releases/download/r608383e/citra-linux-appimage-20240927-608383e.tar.gz" "citra" "tar.gz" "emulator" "$showProgress"; then #citra-qt.AppImage #if installEmuAI "$Citra_emuName" "" "https://github.com/PabloMK7/citra/releases/download/r518f723/citra-linux-appimage-20240717-518f723.tar.gz" "citra" "tar.gz" "emulator" "$showProgress"; then #citra-qt.AppImage mkdir "$HOME/Applications/citra-temp" tar -xvzf "$HOME/Applications/citra.tar.gz" -C "$HOME/Applications/citra-temp" --strip-components 1 From 3c3d093424eb76e126cc59769a0f9d5d19579e02 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 18:27:45 +0100 Subject: [PATCH 037/230] BigPEmu linux --- functions/EmuScripts/emuDeckBigPEmu.sh | 13 ++-- tools/launchers/bigpemu.sh | 86 +++++++++++--------------- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/functions/EmuScripts/emuDeckBigPEmu.sh b/functions/EmuScripts/emuDeckBigPEmu.sh index aaa6e6f8f..358d1897d 100644 --- a/functions/EmuScripts/emuDeckBigPEmu.sh +++ b/functions/EmuScripts/emuDeckBigPEmu.sh @@ -17,12 +17,13 @@ BigPEmu_install(){ mkdir -p $BigPEmu_appData - BigPEmudownloadLink=$(curl -s "https://www.richwhitehouse.com/jaguar/index.php?content=download" | grep -o 'https://www\.richwhitehouse\.com/jaguar/builds/BigPEmu_v[0-9]*\.zip' | grep -v "BigPEmu_*-DEV.zip" | head -n 1) + BigPEmudownloadLink=$(curl -s "https://www.richwhitehouse.com/jaguar/index.php?content=download" | grep -o 'https://www\.richwhitehouse\.com/jaguar/builds/BigPEmu_Linux64_v[0-9]*\.tar.gz' | grep -v "BigPEmu_*-DEV.tar.gz" | head -n 1) - if safeDownload "BigPEmu" "$BigPEmudownloadLink" "$HOME/Applications/BigPEmu/BigPEmu.zip" "$showProgress"; then + if safeDownload "BigPEmu" "$BigPEmudownloadLink" "$HOME/Applications/BigPEmu/BigPEmu.tar.gz" "$showProgress"; then - unzip -o "$HOME/Applications/BigPEmu/BigPEmu.zip" -d "$HOME/Applications/BigPEmu" - rm -f "$HOME/Applications/BigPEmu/BigPEmu.zip" + tar -xvzf "$HOME/Applications/BigPEmu/BigPEmu.tar.gz" -C "$HOME/Applications/BigPEmu" --strip-components 1 + + rm -f "$HOME/Applications/BigPEmu/BigPEmu.tar.gz" else return 1 @@ -80,7 +81,7 @@ BigPEmu_addESConfig(){ ESDE_junksettingsFile ESDE_addCustomSystemsFile ESDE_setEmulationFolder - + # Atari Jaguar if [[ $(grep -rnw "$es_systemsFile" -e 'atarijaguar') == "" ]]; then xmlstarlet ed -S --inplace --subnode '/systemList' --type elem --name 'system' \ @@ -166,7 +167,7 @@ BigPEmu_wipeSettings(){ #Uninstall BigPEmu_uninstall(){ - uninstallGeneric $BigPEmu_emuName $BigPEmu_emuPath "" "emulator" + uninstallGeneric $BigPEmu_emuName $BigPEmu_emuPath "" "emulator" } #setABXYstyle diff --git a/tools/launchers/bigpemu.sh b/tools/launchers/bigpemu.sh index 161226b06..c3a8b7622 100755 --- a/tools/launchers/bigpemu.sh +++ b/tools/launchers/bigpemu.sh @@ -1,56 +1,44 @@ -#!/usr/bin/bash -# bigpemu.sh - -# Set up cloud save -source "${HOME}/.config/EmuDeck/backend/functions/all.sh" -emulatorInit "bigpemu" - -# Get SELFPATH -SELFPATH="$( realpath "${BASH_SOURCE[0]}" )" - -# Get EXE -EXE="\"/usr/bin/bash\" \"${SELFPATH}\"" -echo "EXE: ${EXE}" - -# NAME -NAME="BigPEmu" - -# AppID.py -APPIDPY="$emulationPath/tools/appID.py" - -# Proton Launcher Script -PROTONLAUNCH="$emulationPath/tools/proton-launch.sh" - -# BigPEmu location -BIGPEMU="$HOME/Applications/BigPEmu/BigPEmu.exe" - -# APPID -if [ -e "/usr/bin/python" ]; then - APPID=$( /usr/bin/python "${APPIDPY}" "${EXE}" "${NAME}" ) -elif [ -e "/usr/bin/python3" ]; then - APPID=$( /usr/bin/python3 "${APPIDPY}" "${EXE}" "${NAME}" ) -else - echo "Python not found." +#!/bin/bash +source $HOME/.config/EmuDeck/backend/functions/all.sh +emulatorInit "BigPEmu" +emuName="BigPEmu" #parameterize me +emufolder="$HOME/Applications/BigPEmu" # has to be here for ES-DE to find it + +#initialize execute array +exe=() + +#find full path to emu executable +exe_path=$(find "$emufolder" -iname "${emuName}.sh" | sort -n | cut -d' ' -f 2- | tail -n 1 2>/dev/null) + +#if appimage doesn't exist fall back to flatpak. +if [[ -z "$exe_path" ]]; then + #flatpak + flatpakApp=$(flatpak list --app --columns=application | grep "$emuName") + #fill execute array + exe=("flatpak" "run" "$flatpakApp") +else + #make sure that file is executable + chmod +x "$exe_path" + #fill execute array + exe=("$exe_path") fi -echo "APPID: ${APPID}" - -# Proton Version -PROTONVER="8.0" - -# Call the Proton launcher script and give the arguments - -if [ -z "${@}" ]; then +#run the executable with the params. +launch_args=() +for rom in "${@}"; do + # Parsers previously had single quotes ("'/path/to/rom'" ), this allows those shortcuts to continue working. + removedLegacySingleQuotes=$(echo "$rom" | sed "s/^'//; s/'$//") + launch_args+=("$removedLegacySingleQuotes") +done - echo "${PROTONLAUNCH}" -p "${PROTONVER}" -i "${APPID}" -- "${BIGPEMU}" "*" -localdata >> "${LOGFILE}" - "${PROTONLAUNCH}" -p "${PROTONVER}" -i "${APPID}" -- "${BIGPEMU}" "*" -localdata - echo "Launching BigPEmu directly" +echo "Launching: ${exe[*]} ${launch_args[*]}" +if [[ -z "${*}" ]]; then + echo "ROM not found. Launching $emuName directly" + "${exe[@]}" else - - echo "${PROTONLAUNCH}" -p "${PROTONVER}" -i "${APPID}" -- "${BIGPEMU}" "${@}" -localdata >> "${LOGFILE}" - "${PROTONLAUNCH}" -p "${PROTONVER}" -i "${APPID}" -- "${BIGPEMU}" "${@}" -localdata echo "ROM found, launching game" - + "${exe[@]}" "${launch_args[@]}" fi -rm -rf "$savesPath/.gaming" \ No newline at end of file + +rm -rf "$savesPath/.gaming" From b6d6445a4038daaa1275412b882b5ea0678495ce Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 18:27:45 +0100 Subject: [PATCH 038/230] BigPEmu linux --- tools/launchers/bigpemu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/launchers/bigpemu.sh b/tools/launchers/bigpemu.sh index c3a8b7622..233d8f40d 100755 --- a/tools/launchers/bigpemu.sh +++ b/tools/launchers/bigpemu.sh @@ -1,7 +1,7 @@ #!/bin/bash source $HOME/.config/EmuDeck/backend/functions/all.sh emulatorInit "BigPEmu" -emuName="BigPEmu" #parameterize me +emuName="bigpemu" #parameterize me emufolder="$HOME/Applications/BigPEmu" # has to be here for ES-DE to find it #initialize execute array From b1a56efeb480d2112d11c1cfdd58cee133b2f120 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 18:43:26 +0100 Subject: [PATCH 039/230] no proton --- functions/EmuScripts/emuDeckBigPEmu.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/functions/EmuScripts/emuDeckBigPEmu.sh b/functions/EmuScripts/emuDeckBigPEmu.sh index 358d1897d..2a7f390ae 100644 --- a/functions/EmuScripts/emuDeckBigPEmu.sh +++ b/functions/EmuScripts/emuDeckBigPEmu.sh @@ -1,6 +1,6 @@ #!/bin/bash #variables -BigPEmu_emuName="BigPEmu (Proton)" +BigPEmu_emuName="BigPEmu" BigPEmu_emuType="$emuDeckEmuTypeWindows" BigPEmu_emuPath="$HOME/Applications/BigPEmu/BigPEmu.exe" BigPEmu_appData="$HOME/Applications/BigPEmu/UserData" @@ -31,15 +31,15 @@ BigPEmu_install(){ cp "$EMUDECKGIT/tools/launchers/bigpemu.sh" "$toolsPath/launchers/bigpemu.sh" # So users can still open BigPEmu from the ~/Applications folder. - cp "$EMUDECKGIT/tools/launchers/bigpemu.sh" "$HOME/Applications/BigPEmu/bigpemu.sh" + #cp "$EMUDECKGIT/tools/launchers/bigpemu.sh" "$HOME/Applications/BigPEmu/bigpemu.sh" cp "$EMUDECKGIT/tools/launchers/bigpemu.sh" "$romsPath/emulators/bigpemu.sh" chmod +x "${toolsPath}/launchers/bigpemu.sh" - chmod +x "$HOME/Applications/BigPEmu/bigpemu.sh" + #chmod +x "$HOME/Applications/BigPEmu/bigpemu.sh" chmod +x "$romsPath/emulators/bigpemu.sh" - createDesktopShortcut "$HOME/.local/share/applications/BigPEmu (Proton).desktop" \ - "BigPEmu (Proton)" \ + createDesktopShortcut "$HOME/.local/share/applications/BigPEmu.desktop" \ + "BigPEmu" \ "${toolsPath}/launchers/bigpemu.sh -w" \ "False" } @@ -51,7 +51,6 @@ BigPEmu_init(){ BigPEmu_setEmulationFolder BigPEmu_setupSaves BigPEmu_flushEmulatorLauncher - addProtonLaunch #SRM_createParsers if [ -e "$ESDE_toolPath" ] || [ -f "${toolsPath}/$ESDE_downloadedToolName" ] || [ -f "${toolsPath}/$ESDE_oldtoolName.AppImage" ]; then BigPEmu_addESConfig @@ -68,7 +67,6 @@ BigPEmu_update(){ BigPEmu_setEmulationFolder BigPEmu_setupSaves BigPEmu_flushEmulatorLauncher - addProtonLaunch if [ -e "$ESDE_toolPath" ] || [ -f "${toolsPath}/$ESDE_downloadedToolName" ] || [ -f "${toolsPath}/$ESDE_oldtoolName.AppImage" ]; then BigPEmu_addESConfig else @@ -91,7 +89,7 @@ BigPEmu_addESConfig(){ --subnode '$newSystem' --type elem --name 'path' -v '%ROMPATH%/atarijaguar' \ --subnode '$newSystem' --type elem --name 'extension' -v '.abs .ABS .bin .BIN .cdi .CDI .cof .COF .cue .CUE .j64 .J64 .jag .JAG .prg .PRG .rom .ROM .7z .7Z .zip .ZIP' \ --subnode '$newSystem' --type elem --name 'commandB' -v "/usr/bin/bash ${toolsPath}/launchers/bigpemu.sh %ROM%" \ - --insert '$newSystem/commandB' --type attr --name 'label' --value "BigPEmu (Proton)" \ + --insert '$newSystem/commandB' --type attr --name 'label' --value "BigPEmu" \ --subnode '$newSystem' --type elem --name 'commandV' -v "%EMULATOR_RETROARCH% -L %CORE_RETROARCH%/virtualjaguar_libretro.so %ROM%" \ --insert '$newSystem/commandV' --type attr --name 'label' --value "Virtual Jaguar" \ --subnode '$newSystem' --type elem --name 'commandM' -v "%STARTDIR%=~/.mame %EMULATOR_MAME% -rompath %GAMEDIR%\;%ROMPATH%/atarijaguar jaguar -cart %ROM%" \ @@ -116,7 +114,7 @@ BigPEmu_addESConfig(){ --subnode '$newSystem' --type elem --name 'path' -v '%ROMPATH%/atarijaguarcd' \ --subnode '$newSystem' --type elem --name 'extension' -v '.abs .ABS .bin .BIN .cdi .CDI .cof .COF .cue .CUE .j64 .J64 .jag .JAG .prg .PRG .rom .ROM .7z .7Z .zip .ZIP' \ --subnode '$newSystem' --type elem --name 'commandB' -v "/usr/bin/bash ${toolsPath}/launchers/bigpemu.sh %ROM%" \ - --insert '$newSystem/commandB' --type attr --name 'label' --value "BigPEmu (Proton)" \ + --insert '$newSystem/commandB' --type attr --name 'label' --value "BigPEmu" \ --subnode '$newSystem' --type elem --name 'platform' -v 'atarijaguarcd' \ --subnode '$newSystem' --type elem --name 'theme' -v 'atarijaguarcd' \ -r 'systemList/system/commandB' -v 'command' \ From 8eb9130954ce4510b8331d18c93188319ed98100 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 18:44:45 +0100 Subject: [PATCH 040/230] no sh --- tools/launchers/bigpemu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/launchers/bigpemu.sh b/tools/launchers/bigpemu.sh index 233d8f40d..ae5de816c 100755 --- a/tools/launchers/bigpemu.sh +++ b/tools/launchers/bigpemu.sh @@ -8,7 +8,7 @@ emufolder="$HOME/Applications/BigPEmu" # has to be here for ES-DE to find it exe=() #find full path to emu executable -exe_path=$(find "$emufolder" -iname "${emuName}.sh" | sort -n | cut -d' ' -f 2- | tail -n 1 2>/dev/null) +exe_path=$(find "$emufolder" -iname "${emuName}" | sort -n | cut -d' ' -f 2- | tail -n 1 2>/dev/null) #if appimage doesn't exist fall back to flatpak. if [[ -z "$exe_path" ]]; then From 6aca87672cf86f394027c32d9ae2910ae5982784 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 18:46:08 +0100 Subject: [PATCH 041/230] no old launcher --- functions/EmuScripts/emuDeckBigPEmu.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions/EmuScripts/emuDeckBigPEmu.sh b/functions/EmuScripts/emuDeckBigPEmu.sh index 2a7f390ae..1e74484cc 100644 --- a/functions/EmuScripts/emuDeckBigPEmu.sh +++ b/functions/EmuScripts/emuDeckBigPEmu.sh @@ -38,6 +38,9 @@ BigPEmu_install(){ #chmod +x "$HOME/Applications/BigPEmu/bigpemu.sh" chmod +x "$romsPath/emulators/bigpemu.sh" + rm -rf "$HOME/.local/share/applications/BigPEmu.desktop" + rm -rf "$HOME/Applications/BigPEmu/bigpemu.sh" + createDesktopShortcut "$HOME/.local/share/applications/BigPEmu.desktop" \ "BigPEmu" \ "${toolsPath}/launchers/bigpemu.sh -w" \ From 2ef07ce0e9d07f3e5c016a2c059d340a85f4f652 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 12 Nov 2024 18:48:01 +0100 Subject: [PATCH 042/230] FML --- functions/EmuScripts/emuDeckBigPEmu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckBigPEmu.sh b/functions/EmuScripts/emuDeckBigPEmu.sh index 1e74484cc..31951eb63 100644 --- a/functions/EmuScripts/emuDeckBigPEmu.sh +++ b/functions/EmuScripts/emuDeckBigPEmu.sh @@ -38,7 +38,7 @@ BigPEmu_install(){ #chmod +x "$HOME/Applications/BigPEmu/bigpemu.sh" chmod +x "$romsPath/emulators/bigpemu.sh" - rm -rf "$HOME/.local/share/applications/BigPEmu.desktop" + rm -rf "$HOME/.local/share/applications/BigPEmu (Proton).desktop" rm -rf "$HOME/Applications/BigPEmu/bigpemu.sh" createDesktopShortcut "$HOME/.local/share/applications/BigPEmu.desktop" \ From 7171ac47660d129ce233a67a7fc391223f3825d4 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 13:11:26 +0100 Subject: [PATCH 043/230] no fuzzy --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 52d3078ba..fc57241e9 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -100,7 +100,7 @@ generateGameLists_artwork() { #game_name=$(echo "$response" | jq -r '.name') #game_img_url=$(echo "$response" | jq -r '.img') - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg.php?name=$fuzzygame" + wget -q -O "$HOME/emudeck/cache/response.json" 'https://bot.emudeck.com/steamdbimg.php?name="'${game}'"' game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") From 97a9018aaf67dddef5904121dea6f11b7e555b52 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 15:19:23 +0100 Subject: [PATCH 044/230] new parser retrolibrary --- functions/generateGameLists.sh | 156 +----------------- tools/retro-library/download_art.py | 76 +++++++++ tools/{ => retro-library}/fuzzy_search_rom.py | 0 .../generate_game_lists.py | 0 tools/retro-library/missing_artwork.py | 113 +++++++++++++ 5 files changed, 195 insertions(+), 150 deletions(-) create mode 100644 tools/retro-library/download_art.py rename tools/{ => retro-library}/fuzzy_search_rom.py (100%) rename tools/{ => retro-library}/generate_game_lists.py (100%) create mode 100644 tools/retro-library/missing_artwork.py diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index fc57241e9..427538e4a 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -2,33 +2,14 @@ generateGameLists() { pegasus_setPaths mkdir -p "$HOME/emudeck/cache/" - python $HOME/.config/EmuDeck/backend/tools/generate_game_lists.py "$romsPath" + python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" } generateGameListsJson() { - #local userid=$1 -# -# if [ ! -f "$HOME/emudeck/games.json" ]; then -# download_attempt=$(wget --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36" "https://steamgriddb.com/api/games" -O "$HOME/emudeck/games.json") -# fi - python $HOME/.config/EmuDeck/backend/tools/generate_game_lists.py "$romsPath" + python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" cat $HOME/emudeck/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & - if [ -f "$HOME/emudeck/cache/.romlibrary_first" ]; then - generateGameLists_artwork 0 &> /dev/null & - else - generateGameLists_artwork 1 &> /dev/null & - sleep 5 - generateGameLists_artwork 2 &> /dev/null & - sleep 5 - generateGameLists_artwork 3 &> /dev/null & - sleep 5 - generateGameLists_artwork 4 &> /dev/null & - sleep 5 - generateGameLists_artwork 5 &> /dev/null & - sleep 5 - touch "$HOME/emudeck/cache/.romlibrary_first" - fi + generateGameLists_artwork &> /dev/null & } @@ -36,141 +17,16 @@ generateGameLists_importESDE() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/emudeck/" - python $HOME/.config/EmuDeck/backend/tools/retroLibrary/import_media.py "$romsPath" "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/import_media.py "$romsPath" "$dest_folder" } generateGameLists_artwork() { mkdir -p "$HOME/emudeck/cache/" - - local number_log=$1 - local current_time=$(date +"%H_%M_%S") - local logfilename="$HOME/emudeck/logs/library_${number_log}.log" - local json_file="$HOME/emudeck/cache/roms_games.json" - local json=$(cat "$json_file") - - local PIDFILE="$HOME/.config/EmuDeck/retrolibrary_$number_log.pid" - - if [ -f "$PIDFILE" ]; then - exit 1 - else - echo $$ > "$PIDFILE" - if [ $? -ne 0 ]; then - echo "Could not create PID file" - exit 1 - fi - fi - - if [ "$number_log" = 1 ]; then - local platforms=$(echo "$json" | jq -r '.[].id') - else - local platforms=$(echo "$json" | jq -r '.[].id' | shuf) - fi local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/emudeck/" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" - echo "" > "$logfilename" - mkdir -p "$dest_folder" - - declare -A processed_games - - for platform in $platforms; do - echo "Processing platform: $platform" >> "$logfilename" - if [ $number_log = 1 ]; then - games=$(echo "$json" | jq -r --arg platform "$platform" '.[] | select(.id == $platform) | .games[]?.name') - else - games=$(echo "$json" | jq -r --arg platform "$platform" '.[] | select(.id == $platform) | .games[]?.name' | shuf) - fi - declare -a download_array - declare -a download_dest_paths - - for game in $games; do - file_to_check="$dest_folder${game// /_}.jpg" - - if ! ls $file_to_check 1> /dev/null 2>&1 && [ -z "${processed_games[$game]}" ]; then - echo "GAME:" "$game" >> "$logfilename" - - fuzzygame=$(python $HOME/.config/EmuDeck/backend/tools/fuzzy_search_rom.py "$game" "$dest_folder") - fuzzygame="${fuzzygame// /_}" - fuzzygame="${fuzzygame//:/}" - fuzzygame="${fuzzygame//./}" - fuzzygame="${fuzzygame//&/}" - fuzzygame="${fuzzygame//!/}" - echo "FUZZY:" "$fuzzygame" >> "$logfilename" - #response=$(curl -s -G "https://bot.emudeck.com/steamdbimg.php?name=$fuzzygame") - #game_name=$(echo "$response" | jq -r '.name') - #game_img_url=$(echo "$response" | jq -r '.img') - - wget -q -O "$HOME/emudeck/cache/response.json" 'https://bot.emudeck.com/steamdbimg.php?name="'${game}'"' - game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") - game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") - - filename=$(basename "$game_img_url") - dest_path="$dest_folder$game.jpg" - - if [ ! -f "$dest_path" ] && [ "$game_img_url" != "null" ]; then - echo "Added to the list: $game_img_url" >> "$logfilename" - download_array+=("$game_img_url") - download_dest_paths+=("$dest_path") - processed_games[$game]=1 - else - #response=$(curl -s -G "https://bot.emudeck.com/steamdbimg.php?name=$game") - #game_name=$(echo "$response" | jq -r '.name') - #game_img_url=$(echo "$response" | jq -r '.img') - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdbimg.php?name=$game" - - game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") - game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") - filename=$(basename "$game_img_url") - dest_path="$dest_folder$game.jpg" - - if [ "$game_img_url" != "null" ]; then - echo "Added to the list (NO FUZZY): $game_img_url" >> "$logfilename" - download_array+=("$game_img_url") - download_dest_paths+=("$dest_path") - processed_games[$game]=1 - else - echo -e " - No picture" >> "$logfilename" - fi - fi - fi - - # Download in batches of 10 - if [ ${#download_array[@]} -ge 10 ]; then - echo "" - echo "Start batch" >> "$logfilename" - for i in "${!download_array[@]}"; do - { - #curl -s -o "${download_dest_paths[$i]}" "${download_array[$i]}" >> "$logfilename" - wget -q -O "${download_dest_paths[$i]}" "${download_array[$i]}" >> "$logfilename" - - } & - done - wait - # Clear the arrays for the next batch - download_array=() - download_dest_paths=() - echo "Completed batch" >> "$logfilename" - echo "" - fi - done - - # Download images for the current platform - if [ ${#download_array[@]} -ne 0 ]; then - for i in "${!download_array[@]}"; do - { - #curl -s -o "${download_dest_paths[$i]}" "${download_array[$i]}" >> "$logfilename" - wget -q -O "${download_dest_paths[$i]}" "${download_array[$i]}" >> "$logfilename" - } & - done - fi - wait - - echo "Completed search for platform: $platform" >> "$logfilename" - done - - # Save the updated JSON back to the file - #echo "$json" > "$json_file" - rm -rf "$PIDFILE" } saveImage(){ diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py new file mode 100644 index 000000000..6ec73f562 --- /dev/null +++ b/tools/retro-library/download_art.py @@ -0,0 +1,76 @@ +import json +import requests +import os +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed + +# Path del JSON y carpeta de destino desde los argumentos +save_folder = sys.argv[1] +number = sys.argv[2] +json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork{number}.json') + +def create_empty_image(name, save_folder): + # Crear la carpeta si no existe + os.makedirs(save_folder, exist_ok=True) + # Definir la ruta de guardado para el archivo vacío + img_path = os.path.join(save_folder, f"{name}.jpg") + + # Crear un archivo vacío + with open(img_path, 'wb') as file: + pass # No escribimos nada para que quede vacío + print(f"Archivo vacío creado para {name}") + +def download_image(name, img_url, save_folder): + # Crear la carpeta si no existe + os.makedirs(save_folder, exist_ok=True) + # Definir la ruta de guardado + img_path = os.path.join(save_folder, f"{name}.jpg") + + # Descargar y guardar la imagen + response = requests.get(img_url) + if response.status_code == 200: + with open(img_path, 'wb') as file: + file.write(response.content) + print(f"Imagen guardada como {img_path}") + else: + print(f"Error al descargar la imagen para {name}") + +def fetch_image_data(game): + name = game['name'] + url = f"https://bot.emudeck.com/steamdbimg.php?name={name}" + + try: + response = requests.get(url) + if response.status_code == 200: + data = response.json() + img_url = data.get('img') # Usar get para evitar errores si 'img' no existe o es None + if img_url: + download_image(name, img_url, save_folder) + else: + print(f"No se encontró una URL de imagen válida para {name}. Creando archivo vacío.") + create_empty_image(name, save_folder) + else: + print(f"Error al llamar al servicio para {name}") + create_empty_image(name, save_folder) + except requests.RequestException as e: + print(f"Excepción al procesar {name}: {e}") + create_empty_image(name, save_folder) + +def process_json(save_folder): + # Leer el JSON original + with open(json_path, 'r') as file: + games = json.load(file) + + # Usar ThreadPoolExecutor para descargar en paralelo + with ThreadPoolExecutor(max_workers=15) as executor: + futures = [executor.submit(fetch_image_data, game) for game in games] + for future in as_completed(futures): + future.result() # Esperar a que cada tarea termine + +if __name__ == "__main__": + # Verificar que se pasaron los argumentos necesarios + if len(sys.argv) != 3: + print("Uso: python script.py ") + sys.exit(1) + + process_json(save_folder) diff --git a/tools/fuzzy_search_rom.py b/tools/retro-library/fuzzy_search_rom.py similarity index 100% rename from tools/fuzzy_search_rom.py rename to tools/retro-library/fuzzy_search_rom.py diff --git a/tools/generate_game_lists.py b/tools/retro-library/generate_game_lists.py similarity index 100% rename from tools/generate_game_lists.py rename to tools/retro-library/generate_game_lists.py diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py new file mode 100644 index 000000000..32f152f7e --- /dev/null +++ b/tools/retro-library/missing_artwork.py @@ -0,0 +1,113 @@ +import os +import json +import sys +import re + +def generate_game_lists(roms_path, images_path, number): + def collect_game_data(system_dir, extensions): + game_data = [] + for root, _, files in os.walk(system_dir): + for file in files: + file_path = os.path.join(root, file) + if os.path.islink(file_path): + continue + + filename = os.path.basename(file) + extension = filename.split('.')[-1] + name = '.'.join(filename.split('.')[:-1]) + if extension in extensions: + if "wiiu" in system_dir: + if extension == "wux": + name = name + elif extension == "wua": + name = name + else: + parts = root.split(os.sep) + if len(parts) >= 2: + name = parts[-2] + else: + name = name + + if name == "roms": + name = name + + if name == "wiiu": + name = parts[-1] + + if "ps3" in system_dir: + parts = root.split(os.sep) + if len(parts) >= 3: + name = parts[-3] + else: + name = name + + if name == "roms": + name = name + + if name == "ps3": + name = parts[-1] + + platform = os.path.basename(system_dir) + + name_cleaned = re.sub(r'\(.*?\)', '', name) + name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) + name_cleaned = name_cleaned.strip() + name_cleaned = name_cleaned.replace(' ', '_').replace('-', '_') + name_cleaned = re.sub(r'_+', '_', name_cleaned) + name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') + + game_img = f"/customimages/emudeck/{name_cleaned}.jpg" + + # Verificar si la imagen existe en el images_path + + img_path = os.path.join(images_path, f"{name_cleaned}.jpg") + #print(img_path) + if not os.path.exists(img_path): + game_info = { + "name": name_cleaned, + "platform": platform + } + game_data.append(game_info) + + game_data_sorted = sorted(game_data, key=lambda x: x['name']) + return game_data_sorted + + roms_dir = roms_path + valid_system_dirs = [] + + for system_dir in os.listdir(roms_dir): + if system_dir == "xbox360": + system_dir = "xbox360/roms" + full_path = os.path.join(roms_dir, system_dir) + if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): + file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) + if file_count > 2: + valid_system_dirs.append(full_path) + + game_list = [] + + for system_dir in valid_system_dirs: + if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): + continue + + with open(os.path.join(system_dir, 'metadata.txt')) as f: + metadata = f.read() + extensions = next((line.split(':')[1].strip().replace(',', ' ') for line in metadata.splitlines() if line.startswith('extensions:')), '').split() + + games = collect_game_data(system_dir, extensions) + if games: + game_list.extend(games) + + json_output = json.dumps(game_list, indent=4) + home_directory = os.path.expanduser("~") + output_file = os.path.join(home_directory, 'emudeck', 'cache', f'missing_artwork{number}.json') + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'w') as f: + f.write(json_output) + +# Pasar la ruta de las ROMs y de las imágenes desde los argumentos de línea de comandos +roms_path = sys.argv[1] +images_path = sys.argv[2] +number = sys.argv[3] + +generate_game_lists(roms_path, images_path, number) From d723d89c91e96cd3435c87dd344e1fffe72eab30 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 15:21:29 +0100 Subject: [PATCH 045/230] no number --- tools/retro-library/download_art.py | 3 +-- tools/retro-library/missing_artwork.py | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 6ec73f562..9bec28deb 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -6,8 +6,7 @@ # Path del JSON y carpeta de destino desde los argumentos save_folder = sys.argv[1] -number = sys.argv[2] -json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork{number}.json') +json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork.json') def create_empty_image(name, save_folder): # Crear la carpeta si no existe diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 32f152f7e..ef8a5af0e 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -3,7 +3,7 @@ import sys import re -def generate_game_lists(roms_path, images_path, number): +def generate_game_lists(roms_path, images_path): def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -100,7 +100,7 @@ def collect_game_data(system_dir, extensions): json_output = json.dumps(game_list, indent=4) home_directory = os.path.expanduser("~") - output_file = os.path.join(home_directory, 'emudeck', 'cache', f'missing_artwork{number}.json') + output_file = os.path.join(home_directory, 'emudeck', 'cache', f'missing_artwork.json') os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) @@ -108,6 +108,5 @@ def collect_game_data(system_dir, extensions): # Pasar la ruta de las ROMs y de las imágenes desde los argumentos de línea de comandos roms_path = sys.argv[1] images_path = sys.argv[2] -number = sys.argv[3] -generate_game_lists(roms_path, images_path, number) +generate_game_lists(roms_path, images_path) From 52edf6a1368d8617b2897d9fde1e0568d1b4cf22 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 15:22:48 +0100 Subject: [PATCH 046/230] no check --- tools/retro-library/download_art.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 9bec28deb..88201bc3c 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -66,10 +66,4 @@ def process_json(save_folder): for future in as_completed(futures): future.result() # Esperar a que cada tarea termine -if __name__ == "__main__": - # Verificar que se pasaron los argumentos necesarios - if len(sys.argv) != 3: - print("Uso: python script.py ") - sys.exit(1) - process_json(save_folder) From cf8e27d5fa9fff7e5dd1af852b14baaac40e140e Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 15:43:24 +0100 Subject: [PATCH 047/230] no need for number --- tools/retro-library/download_art.py | 2 +- tools/retro-library/missing_artwork.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 88201bc3c..e9219707e 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -66,4 +66,4 @@ def process_json(save_folder): for future in as_completed(futures): future.result() # Esperar a que cada tarea termine - process_json(save_folder) +process_json(save_folder) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index ef8a5af0e..9f1b3e2ae 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -100,7 +100,7 @@ def collect_game_data(system_dir, extensions): json_output = json.dumps(game_list, indent=4) home_directory = os.path.expanduser("~") - output_file = os.path.join(home_directory, 'emudeck', 'cache', f'missing_artwork.json') + output_file = os.path.join(home_directory, 'emudeck', 'cache', 'missing_artwork.json') os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) From 1d075fd574eb3577d9688e3c91a98467133d0195 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 16:30:03 +0100 Subject: [PATCH 048/230] percentage --- functions/generateGameLists.sh | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 427538e4a..e84909ab3 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -56,10 +56,21 @@ function addGameListsArtwork() { generateGameLists_getPercentage() { local json_file="$HOME/emudeck/cache/roms_games.json" - local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local destination="$accountfolder/config/grid/emudeck" - local jpg_count=$(find "$destination" -type f -name "*.jpg" | wc -l) + local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" + local games=$(jq '[.[].games[]] | length' "$json_file") - local percentage=$(( 100 * jpg_count / games )) - echo "$jpg_count / $games ($percentage%)" + + local artwork_missing=$(jq '.[] | length' "$json_file_artwork") + + # Verificar que games no sea cero para evitar división por cero + if [ "$games" -eq 0 ]; then + echo "No se encontraron juegos en $json_file" + echo "0 / 0" + fi + + local parsed_games=$(( games - artwork_missing )) + + local percentage=$(( 100 * parsed_games / games )) + + echo "$parsed_games / $games ($percentage%)" } \ No newline at end of file From b3ee4f9bc704c59184311245f98e7e4d98cbc69b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 16:34:51 +0100 Subject: [PATCH 049/230] spaces --- functions/generateGameLists.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index e84909ab3..f26128a41 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -59,18 +59,15 @@ generateGameLists_getPercentage() { local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" local games=$(jq '[.[].games[]] | length' "$json_file") - local artwork_missing=$(jq '.[] | length' "$json_file_artwork") - # Verificar que games no sea cero para evitar división por cero + if [ "$games" -eq 0 ]; then - echo "No se encontraron juegos en $json_file" echo "0 / 0" + exit fi local parsed_games=$(( games - artwork_missing )) - local percentage=$(( 100 * parsed_games / games )) - echo "$parsed_games / $games ($percentage%)" } \ No newline at end of file From 2af724002887d537e9b9a3a907edbb5f55478338 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 16:39:19 +0100 Subject: [PATCH 050/230] json output --- functions/generateGameLists.sh | 4 ++-- tools/retro-library/generate_game_lists.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index f26128a41..383133df0 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -7,7 +7,7 @@ generateGameLists() { generateGameListsJson() { python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" - cat $HOME/emudeck/cache/roms_games.json + #cat $HOME/emudeck/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & generateGameLists_artwork &> /dev/null & @@ -59,7 +59,7 @@ generateGameLists_getPercentage() { local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" local games=$(jq '[.[].games[]] | length' "$json_file") - local artwork_missing=$(jq '.[] | length' "$json_file_artwork") + local artwork_missing=$(jq '[] | length' "$json_file_artwork") if [ "$games" -eq 0 ]; then diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 76c172e12..a9747fee2 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -121,6 +121,7 @@ def collect_game_data(system_dir, extensions): os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) + print(json_output) roms_path = sys.argv[1] From f2055d464b7f27ad4e1da9f05d590ba187fb0822 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 16:41:10 +0100 Subject: [PATCH 051/230] missing art --- functions/generateGameLists.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 383133df0..4497c3462 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -55,6 +55,9 @@ function addGameListsArtwork() { } generateGameLists_getPercentage() { + + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + local json_file="$HOME/emudeck/cache/roms_games.json" local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" From f72511e626bb3431f91ce61b0aec0f84071a7172 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 16:43:20 +0100 Subject: [PATCH 052/230] . fix --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 4497c3462..17fce9dbb 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -62,7 +62,7 @@ generateGameLists_getPercentage() { local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" local games=$(jq '[.[].games[]] | length' "$json_file") - local artwork_missing=$(jq '[] | length' "$json_file_artwork") + local artwork_missing=$(jq '. | length' "$json_file_artwork") if [ "$games" -eq 0 ]; then From 115fed9deeae32fc346b03f28072db98c87b1407 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 16:51:00 +0100 Subject: [PATCH 053/230] fml --- functions/generateGameLists.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 17fce9dbb..44a2104dc 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -55,22 +55,22 @@ function addGameListsArtwork() { } generateGameLists_getPercentage() { - - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" - local json_file="$HOME/emudeck/cache/roms_games.json" local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" + # Contar el número total de juegos en `roms_games.json` local games=$(jq '[.[].games[]] | length' "$json_file") - local artwork_missing=$(jq '. | length' "$json_file_artwork") + local artwork_missing=$(jq '. | length' "$json_file_artwork") - if [ "$games" -eq 0 ]; then - echo "0 / 0" - exit + if [[ -z "$games" || "$games" -eq 0 ]]; then + return fi local parsed_games=$(( games - artwork_missing )) + local percentage=$(( 100 * parsed_games / games )) + + # Mostrar el resultado echo "$parsed_games / $games ($percentage%)" } \ No newline at end of file From e8e98c9723eb25b0158b7d563aac9a5ccbe235fa Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 16:52:48 +0100 Subject: [PATCH 054/230] missing artwork refresh --- functions/generateGameLists.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 44a2104dc..f7f3e50ac 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -55,6 +55,10 @@ function addGameListsArtwork() { } generateGameLists_getPercentage() { + + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + + local json_file="$HOME/emudeck/cache/roms_games.json" local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" From d49d2d2b4d30da62d04745c9fc7b1e3a7089aa02 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 13 Nov 2024 17:48:59 +0100 Subject: [PATCH 055/230] missing var --- functions/generateGameLists.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index f7f3e50ac..47c4d78ca 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -56,6 +56,9 @@ function addGameListsArtwork() { generateGameLists_getPercentage() { + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) + local dest_folder="$accountfolder/config/grid/emudeck/" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" From 6b5d98f55094c8aad85f16e73cdb62da1ce483b4 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 08:38:46 +0100 Subject: [PATCH 056/230] platofrm --- functions/generateGameLists.sh | 6 ++++-- tools/retro-library/generate_game_lists.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 47c4d78ca..f440a256d 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -32,8 +32,9 @@ generateGameLists_artwork() { saveImage(){ local url=$1 local name=$2 + local system=$3 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/emudeck" + local dest_folder="$accountfolder/config/grid/emudeck/${system}" local dest_path="$dest_folder/$name.jpg" wget -q -O "$dest_path" "$url" } @@ -41,8 +42,9 @@ saveImage(){ function addGameListsArtwork() { local file="$1" local appID="$2" + local platform="$2" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local origin="$accountfolder/config/grid/emudeck/$file.jpg" + local origin="$accountfolder/config/grid/emudeck/${platform}/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" local destination_home="$accountfolder/config/grid/${appID}.png" diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index a9747fee2..fdd33546b 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -64,11 +64,11 @@ def collect_game_data(system_dir, extensions): name_cleaned_pegasus = name.replace(',_', ',') clean_name = name_cleaned - game_img = f"/customimages/emudeck/{clean_name}.jpg" + game_img = f"/customimages/emudeck/{platform}/{clean_name}.jpg" game_info = { "name": clean_name, "filename": file_path, - "file": name_cleaned_pegasus, + "file": clean_name, "img": game_img, "platform": platform } From c003cdd8c47899f78254a23a9061338facd6f085 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 08:47:46 +0100 Subject: [PATCH 057/230] platform fixes --- tools/retro-library/download_art.py | 27 +++++++++++++------------- tools/retro-library/missing_artwork.py | 4 ++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index e9219707e..7f645fbda 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -8,22 +8,22 @@ save_folder = sys.argv[1] json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork.json') -def create_empty_image(name, save_folder): +def create_empty_image(name, platform, save_folder): # Crear la carpeta si no existe os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado para el archivo vacío - img_path = os.path.join(save_folder, f"{name}.jpg") + img_path = os.path.join(save_folder, f"{platform}/{name}.jpg") # Crear un archivo vacío with open(img_path, 'wb') as file: pass # No escribimos nada para que quede vacío print(f"Archivo vacío creado para {name}") -def download_image(name, img_url, save_folder): +def download_image(name, platform, img_url, save_folder): # Crear la carpeta si no existe os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado - img_path = os.path.join(save_folder, f"{name}.jpg") + img_path = os.path.join(save_folder, f"{platform}/{name}.jpg") # Descargar y guardar la imagen response = requests.get(img_url) @@ -32,11 +32,12 @@ def download_image(name, img_url, save_folder): file.write(response.content) print(f"Imagen guardada como {img_path}") else: - print(f"Error al descargar la imagen para {name}") + print(f"Error al descargar la imagen para {platform}/{name}") def fetch_image_data(game): name = game['name'] - url = f"https://bot.emudeck.com/steamdbimg.php?name={name}" + platform = game['platform'] + url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}" try: response = requests.get(url) @@ -44,16 +45,16 @@ def fetch_image_data(game): data = response.json() img_url = data.get('img') # Usar get para evitar errores si 'img' no existe o es None if img_url: - download_image(name, img_url, save_folder) + download_image(name, platform, img_url, save_folder) else: - print(f"No se encontró una URL de imagen válida para {name}. Creando archivo vacío.") - create_empty_image(name, save_folder) + print(f"No se encontró una URL de imagen válida para {platform}/{name}. Creando archivo vacío.") + create_empty_image(name, platform, save_folder) else: - print(f"Error al llamar al servicio para {name}") - create_empty_image(name, save_folder) + print(f"Error al llamar al servicio para {platform}/{name}") + create_empty_image(name, platform, save_folder) except requests.RequestException as e: - print(f"Excepción al procesar {name}: {e}") - create_empty_image(name, save_folder) + print(f"Excepción al procesar {platform}/{name}: {e}") + create_empty_image(name, platform, save_folder) def process_json(save_folder): # Leer el JSON original diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 9f1b3e2ae..609e5ac67 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -56,11 +56,11 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - game_img = f"/customimages/emudeck/{name_cleaned}.jpg" + game_img = f"/customimages/emudeck/{platform}/{name_cleaned}.jpg" # Verificar si la imagen existe en el images_path - img_path = os.path.join(images_path, f"{name_cleaned}.jpg") + img_path = os.path.join(images_path, f"{platform}/{name_cleaned}.jpg") #print(img_path) if not os.path.exists(img_path): game_info = { From 6ecd264183d53b9535f3c6308ae21f164b8042e7 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 08:57:28 +0100 Subject: [PATCH 058/230] mkdir --- functions/generateGameLists.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index f440a256d..deb87743d 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -1,6 +1,10 @@ #!/bin/bash generateGameLists() { + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) + local dest_folder="$accountfolder/config/grid/emudeck/" + mkdir -p $dest_folder pegasus_setPaths + rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$dest_folder" --keep-dirlinks mkdir -p "$HOME/emudeck/cache/" python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" } @@ -14,8 +18,7 @@ generateGameListsJson() { } generateGameLists_importESDE() { - local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/emudeck/" + python $HOME/.config/EmuDeck/backend/tools/retro-library/import_media.py "$romsPath" "$dest_folder" } From 4bbb57c1b41732a576c10786db3b5f00c03affbc Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 08:57:36 +0100 Subject: [PATCH 059/230] print dl --- tools/retro-library/download_art.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 7f645fbda..a04207574 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -13,7 +13,7 @@ def create_empty_image(name, platform, save_folder): os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado para el archivo vacío img_path = os.path.join(save_folder, f"{platform}/{name}.jpg") - + print(img_path) # Crear un archivo vacío with open(img_path, 'wb') as file: pass # No escribimos nada para que quede vacío @@ -24,7 +24,7 @@ def download_image(name, platform, img_url, save_folder): os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado img_path = os.path.join(save_folder, f"{platform}/{name}.jpg") - + print(img_path) # Descargar y guardar la imagen response = requests.get(img_url) if response.status_code == 200: From 33704b4f623a31f7ccdc94e818536e61f61e986b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 17:14:54 +0100 Subject: [PATCH 060/230] 3 --- functions/generateGameLists.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index deb87743d..4c48842e0 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -45,7 +45,7 @@ saveImage(){ function addGameListsArtwork() { local file="$1" local appID="$2" - local platform="$2" + local platform="$3" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local origin="$accountfolder/config/grid/emudeck/${platform}/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" @@ -54,9 +54,9 @@ function addGameListsArtwork() { rm -rf "$destination" rm -rf "$destination_hero" rm -rf "$destination_home" - cp -rf "$origin" "$destination" - cp -rf "$origin" "$destination_hero" - cp -rf "$origin" "$destination_home" + ln -s "$origin" "$destination" + ln -s "$origin" "$destination_hero" + ln -s "$origin" "$destination_home" } generateGameLists_getPercentage() { From a07650ab9d61df345d749bbd9730ebdd91d9b40d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 17:35:07 +0100 Subject: [PATCH 061/230] extra artwork --- functions/generateGameLists.sh | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 4c48842e0..89785f8dd 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -47,16 +47,19 @@ function addGameListsArtwork() { local appID="$2" local platform="$3" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local origin="$accountfolder/config/grid/emudeck/${platform}/$file.jpg" + + local tempGrid =$(generateGameLists_extraArtwork $file $platform) + + local origin="$tempGrid" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" local destination_home="$accountfolder/config/grid/${appID}.png" rm -rf "$destination" rm -rf "$destination_hero" rm -rf "$destination_home" - ln -s "$origin" "$destination" - ln -s "$origin" "$destination_hero" - ln -s "$origin" "$destination_home" + cp "$origin" "$destination" + cp "$origin" "$destination_hero" + cp "$origin" "$destination_home" } generateGameLists_getPercentage() { @@ -85,4 +88,23 @@ generateGameLists_getPercentage() { # Mostrar el resultado echo "$parsed_games / $games ($percentage%)" +} + +generateGameLists_extraArtwork() { + local game=$1 + local platform=$2 + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) + local dest_folder="$accountfolder/config/grid/emudeck" + + wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game" + + game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") + game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") + filename=$(basename "$game_img_url") + dest_path="$dest_folder/$platform/$game.temp.jpg" + + if [ "$game_img_url" != "null" ]; then + wget -q -O "${dest_path}" "${game_img_url}" + fi + echo $dest_path } \ No newline at end of file From b9d657da271eef7a9f84dca209f66552518718a6 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 17:37:39 +0100 Subject: [PATCH 062/230] proper img --- functions/generateGameLists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 89785f8dd..9efc42e83 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -50,14 +50,14 @@ function addGameListsArtwork() { local tempGrid =$(generateGameLists_extraArtwork $file $platform) - local origin="$tempGrid" + local origin="$accountfolder/config/grid/emudeck/${platform}/$file.jpg" local destination="$accountfolder/config/grid/${appID}p.png" local destination_hero="$accountfolder/config/grid/${appID}_hero.png" local destination_home="$accountfolder/config/grid/${appID}.png" rm -rf "$destination" rm -rf "$destination_hero" rm -rf "$destination_home" - cp "$origin" "$destination" + cp "$tempGrid" "$destination" cp "$origin" "$destination_hero" cp "$origin" "$destination_home" } From 8eb6ad48655b6d5afa4ecb821b1dd7c17316ea1e Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 18:16:11 +0100 Subject: [PATCH 063/230] recent pictures --- functions/generateGameLists.sh | 37 ++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 9efc42e83..6d3c57f2e 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -48,18 +48,25 @@ function addGameListsArtwork() { local platform="$3" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local tempGrid =$(generateGameLists_extraArtwork $file $platform) + local tempGrid=$(generateGameLists_extraArtwork $file $platform) - local origin="$accountfolder/config/grid/emudeck/${platform}/$file.jpg" - local destination="$accountfolder/config/grid/${appID}p.png" - local destination_hero="$accountfolder/config/grid/${appID}_hero.png" - local destination_home="$accountfolder/config/grid/${appID}.png" - rm -rf "$destination" + # local vertical=$(echo "$tempGrid" | jq -r '.vertical') + local grid=$(echo "$tempGrid" | jq -r '.grid') + vertical="$accountfolder/config/grid/emudeck/$platform/$file.jpg" + + # vertical="$accountfolder/config/grid/emudeck/$platform/$file.grid.temp" + # vertical="$accountfolder/config/grid/emudeck/$platform/$file.grid.temp" #vertical + + local destination_vertical="$accountfolder/config/grid/${appID}p.png" #vertical + local destination_hero="$accountfolder/config/grid/${appID}_hero.png" #BG + local destination_grid="$accountfolder/config/grid/${appID}.png" #GRID + rm -rf "$destination_vertical" rm -rf "$destination_hero" - rm -rf "$destination_home" - cp "$tempGrid" "$destination" - cp "$origin" "$destination_hero" - cp "$origin" "$destination_home" + rm -rf "$destination_grid" + + cp "$vertical" "$destination_vertical" + cp "$grid" "$destination_hero" + cp "$grid" "$destination_grid" } generateGameLists_getPercentage() { @@ -86,7 +93,6 @@ generateGameLists_getPercentage() { local percentage=$(( 100 * parsed_games / games )) - # Mostrar el resultado echo "$parsed_games / $games ($percentage%)" } @@ -99,12 +105,13 @@ generateGameLists_extraArtwork() { wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game" game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") - game_img_url=$(jq -r '.img' "$HOME/emudeck/cache/response.json") - filename=$(basename "$game_img_url") - dest_path="$dest_folder/$platform/$game.temp.jpg" + game_img_url=$(jq -r '.grid' "$HOME/emudeck/cache/response.json") + dest_path="$dest_folder/$platform/$game.grid.temp" if [ "$game_img_url" != "null" ]; then wget -q -O "${dest_path}" "${game_img_url}" fi - echo $dest_path + json=$(jq -n --arg grid "$dest_path" '{grid: $grid}') + + echo "$json" } \ No newline at end of file From 3598c747469bc5ef25f86d4b52932601341ca4b9 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 14 Nov 2024 18:50:52 +0100 Subject: [PATCH 064/230] vertical for everything --- functions/generateGameLists.sh | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 6d3c57f2e..496d6d4a1 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -48,15 +48,12 @@ function addGameListsArtwork() { local platform="$3" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local tempGrid=$(generateGameLists_extraArtwork $file $platform) - - # local vertical=$(echo "$tempGrid" | jq -r '.vertical') - local grid=$(echo "$tempGrid" | jq -r '.grid') - vertical="$accountfolder/config/grid/emudeck/$platform/$file.jpg" - - # vertical="$accountfolder/config/grid/emudeck/$platform/$file.grid.temp" - # vertical="$accountfolder/config/grid/emudeck/$platform/$file.grid.temp" #vertical + #Uncomment to get custom grid + #local tempGrid=$(generateGameLists_extraArtwork $file $platform) + #local grid=$(echo "$tempGrid" | jq -r '.grid') + local vertical="$accountfolder/config/grid/emudeck/$platform/$file.jpg" + local grid=$vertical local destination_vertical="$accountfolder/config/grid/${appID}p.png" #vertical local destination_hero="$accountfolder/config/grid/${appID}_hero.png" #BG local destination_grid="$accountfolder/config/grid/${appID}.png" #GRID @@ -64,9 +61,10 @@ function addGameListsArtwork() { rm -rf "$destination_hero" rm -rf "$destination_grid" - cp "$vertical" "$destination_vertical" - cp "$grid" "$destination_hero" - cp "$grid" "$destination_grid" + #Use CP if custom grid instead of ln.. + ln -s "$vertical" "$destination_vertical" + ln -s "$grid" "$destination_hero" + ln -s "$grid" "$destination_grid" } generateGameLists_getPercentage() { From 176973cbea3f1ccbac65c0919bde642866abfa1d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 15 Nov 2024 21:23:57 +0100 Subject: [PATCH 065/230] 5 workers --- tools/retro-library/download_art.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index a04207574..ec83198c6 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -62,7 +62,7 @@ def process_json(save_folder): games = json.load(file) # Usar ThreadPoolExecutor para descargar en paralelo - with ThreadPoolExecutor(max_workers=15) as executor: + with ThreadPoolExecutor(max_workers=5) as executor: futures = [executor.submit(fetch_image_data, game) for game in games] for future in as_completed(futures): future.result() # Esperar a que cada tarea termine From 6db06200303e8dcbc129446955f404de3d5a0757 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 09:57:30 +0100 Subject: [PATCH 066/230] esde import tool --- tools/retro-library/import_media.py | 87 +++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tools/retro-library/import_media.py diff --git a/tools/retro-library/import_media.py b/tools/retro-library/import_media.py new file mode 100644 index 000000000..01adcf8c8 --- /dev/null +++ b/tools/retro-library/import_media.py @@ -0,0 +1,87 @@ +import os +import re +import shutil +import argparse +from pathlib import Path + +# Configuración de argumentos de línea de comandos +parser = argparse.ArgumentParser(description="Script para copiar archivos de medios normalizados.") +parser.add_argument("roms_dir", type=str, help="Ruta del directorio de roms") +parser.add_argument("target_dir", type=str, help="Ruta del directorio de destino") +args = parser.parse_args() + +# Definir rutas desde los argumentos +roms_dir = Path(args.roms_dir) +target_dir = Path(args.target_dir) + +# Crear la carpeta de destino si no existe +target_dir.mkdir(parents=True, exist_ok=True) + +# Función para leer las extensiones del archivo metadata.txt +def get_extensions_from_metadata(file_path): + extensions = [] + with open(file_path, 'r') as file: + for line in file: + if line.startswith("extensions:"): + ext_string = line.split(":", 1)[1].strip() + extensions = [ext.strip().lower() for ext in ext_string.split(",")] + break + return extensions + +# Función para obtener los archivos de roms con las extensiones especificadas +def get_rom_files(directory, extensions): + rom_files = [] + for file in directory.iterdir(): + if file.is_file() and any(file.name.lower().endswith(ext) for ext in extensions): + rom_files.append(file) # Guardamos la ruta completa del archivo ROM + return rom_files + +# Función para normalizar el nombre del archivo +def normalize_filename(name): + name_cleaned = re.sub(r'\(.*?\)', '', name) + name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) + name_cleaned = name_cleaned.strip() + + name_cleaned = name_cleaned.replace(' ', '_') + name_cleaned = name_cleaned.replace('-', '_') + name_cleaned = re.sub(r'_+', '_', name_cleaned) + name_cleaned = name_cleaned.replace('+', '') + name_cleaned = name_cleaned.replace('&', '') + name_cleaned = name_cleaned.replace('!', '') + name_cleaned = name_cleaned.replace("'", '') + name_cleaned = name_cleaned.replace('.', '') + name_cleaned = name_cleaned.replace(',_', ',') + + return name_cleaned + +# Función para buscar coincidencias en media y copiar el archivo correspondiente con extensión forzada a jpg +def find_and_copy_media(rom_files, media_dir, target_dir): + media_files = {media.stem: media for media in media_dir.iterdir() if media.is_file()} + + for rom in rom_files: + rom_name = rom.stem + # Buscar coincidencia exacta en media (basado en el nombre sin extensión) + if rom_name in media_files: + media_file = media_files[rom_name] # Obtener el archivo de media correspondiente + # Normalizar el nombre del archivo destino + normalized_name = normalize_filename(rom.stem) + target_path = target_dir / rom / f"{normalized_name}.jpg" + shutil.copy(media_file, target_path) + print(f"Copiado: {media_file} -> {target_path}") + +# Iterar por cada subcarpeta de plataforma en el directorio de roms +for platform_dir in roms_dir.iterdir(): + if platform_dir.is_dir() and platform_dir.name != "media": + metadata_file = platform_dir / "metadata.txt" + + if metadata_file.exists(): + extensions = get_extensions_from_metadata(metadata_file) + rom_files = get_rom_files(platform_dir, extensions) + + # Definir el directorio de media específico para la plataforma + media_dir = platform_dir / "media/box2dfront" + + if media_dir.exists(): + find_and_copy_media(rom_files, media_dir, target_dir) + else: + print(f"Advertencia: La carpeta de media {media_dir} no existe para la plataforma {platform_dir.name}.") From b029f4b2fabca42a25954cf777380008df8a009c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 09:58:28 +0100 Subject: [PATCH 067/230] no pip --- install-early.sh | 14 +++++++------- install-unstable.sh | 14 +++++++------- install.sh | 14 +++++++------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/install-early.sh b/install-early.sh index fd4bee355..6da5667d0 100644 --- a/install-early.sh +++ b/install-early.sh @@ -6,13 +6,13 @@ if [ linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi #Python PIP for steamOS -if [ $linuxID == "SteamOS" ]; then - python -m pip --version &> /dev/null || python -m ensurepip --upgrade - python -m pip install --upgrade pip - grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc - grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc - PATH="$HOME/.local/bin:$PATH" -fi +#if [ $linuxID == "SteamOS" ]; then +# python -m pip --version &> /dev/null || python -m ensurepip --upgrade +# python -m pip install --upgrade pip +# grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc +# grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc +# PATH="$HOME/.local/bin:$PATH" +#fi if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" diff --git a/install-unstable.sh b/install-unstable.sh index a0bb29f26..612aec489 100644 --- a/install-unstable.sh +++ b/install-unstable.sh @@ -6,13 +6,13 @@ if [ linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi #Python PIP for steamOS -if [ $linuxID == "SteamOS" ]; then - python -m pip --version &> /dev/null || python -m ensurepip --upgrade - python -m pip install --upgrade pip - grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc - grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc - PATH="$HOME/.local/bin:$PATH" -fi +#if [ $linuxID == "SteamOS" ]; then +# python -m pip --version &> /dev/null || python -m ensurepip --upgrade +# python -m pip install --upgrade pip +# grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc +# grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc +# PATH="$HOME/.local/bin:$PATH" +#fi if [ $linuxID != "ChimeraOS" ]; then echo "installing EmuDeck" diff --git a/install.sh b/install.sh index 527261c0c..34f511ae5 100644 --- a/install.sh +++ b/install.sh @@ -6,13 +6,13 @@ if [ linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi #Python PIP for steamOS -if [ $linuxID == "SteamOS" ]; then - python -m pip --version &> /dev/null || python -m ensurepip --upgrade - python -m pip install --upgrade pip - grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc - grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc - PATH="$HOME/.local/bin:$PATH" -fi +#if [ $linuxID == "SteamOS" ]; then +# python -m pip --version &> /dev/null || python -m ensurepip --upgrade +# python -m pip install --upgrade pip +# grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc +# grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc +# PATH="$HOME/.local/bin:$PATH" +#fi if [ $linuxID != "ChimeraOS" ]; then From 160511b2959254d7e18826da31460d427014289b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 17:56:34 +0100 Subject: [PATCH 068/230] emudeck-cloud 2 --- configs/rclone/rclone.conf | 9 ++++----- functions/ToolScripts/emuDeckCloudSync.sh | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/configs/rclone/rclone.conf b/configs/rclone/rclone.conf index 8b7f90705..a480a29ce 100644 --- a/configs/rclone/rclone.conf +++ b/configs/rclone/rclone.conf @@ -46,8 +46,7 @@ user = pass = [Emudeck-cloud] -type = sftp -host = -user = -pass = -port = \ No newline at end of file +type = b2 +account = +key = +hard_delete = false \ No newline at end of file diff --git a/functions/ToolScripts/emuDeckCloudSync.sh b/functions/ToolScripts/emuDeckCloudSync.sh index 58598d702..031585c7f 100644 --- a/functions/ToolScripts/emuDeckCloudSync.sh +++ b/functions/ToolScripts/emuDeckCloudSync.sh @@ -144,15 +144,15 @@ cloud_sync_setup_providers(){ token="${token//---/|||}" user=$(echo $token | cut -d "|" -f 1) - setSetting cs_user "cs_$user/" + setSetting cs_user "cs$user/" json='{"token":"'"$token"'"}' - password=$(curl --request POST --url "https://token.emudeck.com/create-cs.php" --header "Content-Type: application/json" -d "${json}" | jq -r .cloud_token) - host="cloud.emudeck.com" - port="22" + read cloud_key_id cloud_key < <(curl --request POST --url "https://token.emudeck.com/b2.php" \ + --header "Content-Type: application/json" \ + -d "${json}" | jq -r '.cloud_key_id, .cloud_key') - "$cloud_sync_bin" config update "$cloud_sync_provider" host="$host" user="cs_$user" port="$port" pass="$password" + "$cloud_sync_bin" config update "$cloud_sync_provider" key="$cloud_key" account="$cloud_key_id" "$cloud_sync_bin" mkdir "$cloud_sync_provider:"$cs_user"Emudeck/saves" cloud_sync_save_hash $savesPath From b7a1a419f1cf458db7c2cebaf78564229d569a4a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 17:56:34 +0100 Subject: [PATCH 069/230] emudeck-cloud 2 --- configs/rclone/rclone.conf | 9 ++++----- functions/ToolScripts/emuDeckCloudSync.sh | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/configs/rclone/rclone.conf b/configs/rclone/rclone.conf index 8b7f90705..a480a29ce 100644 --- a/configs/rclone/rclone.conf +++ b/configs/rclone/rclone.conf @@ -46,8 +46,7 @@ user = pass = [Emudeck-cloud] -type = sftp -host = -user = -pass = -port = \ No newline at end of file +type = b2 +account = +key = +hard_delete = false \ No newline at end of file diff --git a/functions/ToolScripts/emuDeckCloudSync.sh b/functions/ToolScripts/emuDeckCloudSync.sh index 58598d702..031585c7f 100644 --- a/functions/ToolScripts/emuDeckCloudSync.sh +++ b/functions/ToolScripts/emuDeckCloudSync.sh @@ -144,15 +144,15 @@ cloud_sync_setup_providers(){ token="${token//---/|||}" user=$(echo $token | cut -d "|" -f 1) - setSetting cs_user "cs_$user/" + setSetting cs_user "cs$user/" json='{"token":"'"$token"'"}' - password=$(curl --request POST --url "https://token.emudeck.com/create-cs.php" --header "Content-Type: application/json" -d "${json}" | jq -r .cloud_token) - host="cloud.emudeck.com" - port="22" + read cloud_key_id cloud_key < <(curl --request POST --url "https://token.emudeck.com/b2.php" \ + --header "Content-Type: application/json" \ + -d "${json}" | jq -r '.cloud_key_id, .cloud_key') - "$cloud_sync_bin" config update "$cloud_sync_provider" host="$host" user="cs_$user" port="$port" pass="$password" + "$cloud_sync_bin" config update "$cloud_sync_provider" key="$cloud_key" account="$cloud_key_id" "$cloud_sync_bin" mkdir "$cloud_sync_provider:"$cs_user"Emudeck/saves" cloud_sync_save_hash $savesPath From f4ca49210ce54f92b61d2c6a503de0e1fce8bd48 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 21:13:34 +0100 Subject: [PATCH 070/230] calculate hash --- functions/generateGameLists.sh | 3 ++- tools/retro-library/download_art.py | 3 ++- tools/retro-library/generate_game_lists.py | 20 ++++++++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 496d6d4a1..980941bac 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -97,10 +97,11 @@ generateGameLists_getPercentage() { generateGameLists_extraArtwork() { local game=$1 local platform=$2 + local hash=$3 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/emudeck" - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game" + wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game&hash=$hash" game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") game_img_url=$(jq -r '.grid' "$HOME/emudeck/cache/response.json") diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index ec83198c6..4b39eb12e 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -37,7 +37,8 @@ def download_image(name, platform, img_url, save_folder): def fetch_image_data(game): name = game['name'] platform = game['platform'] - url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}" + platform = game['hash'] + url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}&hash={hash}" try: response = requests.get(url) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index fdd33546b..dcceeca38 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -2,8 +2,21 @@ import json import sys import re +import hashlib def generate_game_lists(roms_path): + def calculate_hash(file_path): + """Calcula el hash MD5 de un archivo.""" + hash_md5 = hashlib.md5() + try: + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + except Exception as e: + print(f"Error al calcular el hash para {file_path}: {e}") + return None + def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -63,6 +76,9 @@ def collect_game_data(system_dir, extensions): name_cleaned = name_cleaned.replace('.', '') name_cleaned_pegasus = name.replace(',_', ',') + # Calcular el hash de la ROM + rom_hash = calculate_hash(file_path) + clean_name = name_cleaned game_img = f"/customimages/emudeck/{platform}/{clean_name}.jpg" game_info = { @@ -70,7 +86,8 @@ def collect_game_data(system_dir, extensions): "filename": file_path, "file": clean_name, "img": game_img, - "platform": platform + "platform": platform, + "hash": rom_hash # Agregar el hash } game_data.append(game_info) game_data_sorted = sorted(game_data, key=lambda x: x['name']) @@ -114,7 +131,6 @@ def collect_game_data(system_dir, extensions): game_list.append(system_info) game_list_sorted = sorted(game_list, key=lambda x: x['title']) - json_output = json.dumps(game_list_sorted, indent=4) home_directory = os.path.expanduser("~") output_file = os.path.join(home_directory, 'emudeck', 'cache', 'roms_games.json') From f5b11b83bd3a161e1a205c31a1ea99d73ed91bc8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 21:13:53 +0100 Subject: [PATCH 071/230] no need for fuzzy anymore --- tools/retro-library/fuzzy_search_rom.py | 97 ------------------------- 1 file changed, 97 deletions(-) delete mode 100644 tools/retro-library/fuzzy_search_rom.py diff --git a/tools/retro-library/fuzzy_search_rom.py b/tools/retro-library/fuzzy_search_rom.py deleted file mode 100644 index 444f79509..000000000 --- a/tools/retro-library/fuzzy_search_rom.py +++ /dev/null @@ -1,97 +0,0 @@ -import urllib.request -import json -import sys -import os -import subprocess -from difflib import SequenceMatcher -from datetime import datetime, timedelta - -def similar(a, b): - return SequenceMatcher(None, a, b).ratio() - -def simple_similarity(a, b): - # Calcula la longitud de la coincidencia de prefijo más larga - match_length = len([i for i in range(min(len(a), len(b))) if a[i] == b[i]]) - # Normaliza en función de la longitud del título de búsqueda - return match_length / max(len(a), len(b)) - -def find_best_match(search_title, games): - best_match = None - highest_similarity = 0 - - for game in games: - # Calcula la similitud basada en el prefijo - similarity = simple_similarity(search_title, game) - if similarity > highest_similarity: - highest_similarity = similarity - best_match = game - - return best_match - - - - -def is_file_older_than(file_path, days): - file_time = datetime.fromtimestamp(os.path.getmtime(file_path)) - return datetime.now() - file_time > timedelta(days=days) - -# URL del JSON -url = "https://steamgriddb.com/api/games" - -# Título para buscar -search_title = sys.argv[1] -images_path = sys.argv[2] -#print(f"{search_title}") -#sys.exit() - -# Directorio para guardar el archivo JSON -home_dir = os.path.expanduser("~") -emudeck_dir = os.path.join(home_dir, "emudeck") -os.makedirs(emudeck_dir, exist_ok=True) -json_file_path = os.path.join(emudeck_dir, "games.json") - -# Descargar o cargar el JSON -if not os.path.exists(json_file_path) or is_file_older_than(json_file_path, 5): - - bash_command = 'wget --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36" "https://steamgriddb.com/api/games" -O "$HOME/emudeck/games.json"' - try: - result = subprocess.run(bash_command, shell=True, check=True, text=True, capture_output=True) - print("Salida del comando:", result.stdout) - except subprocess.CalledProcessError as e: - print(f"Ocurrió un error al ejecutar el comando: {e}") - -else: - for root, _, files in os.walk(images_path): - for file in files: - file_path = os.path.join(root, file) - # Verifica si el archivo tiene un tamaño de 0 bytes - if os.path.getsize(file_path) == 0: - os.remove(file_path) - - # Cargar el JSON desde el disco duro - with open(json_file_path, "r") as json_file: - data = json_file.read() - -# Intentar cargar el JSON -try: - json_data = json.loads(data) -except json.JSONDecodeError as e: - print(f"Error al decodificar JSON: {e}") - json_data = {} - -# Asegúrate de que esta parte esté adaptada a la estructura real del JSON -if isinstance(json_data, list): - games = [game for game in json_data if "name" in game] -elif isinstance(json_data, dict) and 'games' in json_data: - games = [game for game in json_data['games']] -else: - print("No se encontraron juegos en el JSON o la estructura no es la esperada.") - games = [] - -# Buscar el título más parecido -if games: - best_match = find_best_match(search_title, games) - # Mostrar el resultado - print(f"{best_match}") -else: - print("null") From 78f426da232bfe09435b69579d4733f44cc3f2cc Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 22:10:16 +0100 Subject: [PATCH 072/230] missing hash --- tools/retro-library/missing_artwork.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 609e5ac67..559502e0b 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -2,8 +2,19 @@ import json import sys import re +import hashlib def generate_game_lists(roms_path, images_path): + def calculate_hash(file_path): + hash_md5 = hashlib.md5() + try: + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + except Exception as e: + print(f"Error al calcular el hash para {file_path}: {e}") + return None def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -57,7 +68,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') game_img = f"/customimages/emudeck/{platform}/{name_cleaned}.jpg" - + rom_hash = calculate_hash(file_path) # Verificar si la imagen existe en el images_path img_path = os.path.join(images_path, f"{platform}/{name_cleaned}.jpg") @@ -66,6 +77,7 @@ def collect_game_data(system_dir, extensions): game_info = { "name": name_cleaned, "platform": platform + "hash": rom_hash } game_data.append(game_info) From 25aca4ebc80ba5dea5089cbc12fb6a8515e41724 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 22:10:54 +0100 Subject: [PATCH 073/230] , --- tools/retro-library/missing_artwork.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 559502e0b..0996c2328 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -76,7 +76,7 @@ def collect_game_data(system_dir, extensions): if not os.path.exists(img_path): game_info = { "name": name_cleaned, - "platform": platform + "platform": platform, "hash": rom_hash } game_data.append(game_info) From c1d569208ab69b331a77ac979212786f6cdc4b12 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 22:19:28 +0100 Subject: [PATCH 074/230] hashhash --- tools/retro-library/download_art.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 4b39eb12e..9240ae5f3 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -37,7 +37,7 @@ def download_image(name, platform, img_url, save_folder): def fetch_image_data(game): name = game['name'] platform = game['platform'] - platform = game['hash'] + hash = game['hash'] url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}&hash={hash}" try: From deae4968bbfc6c96819b118ecf247ae4e9855909 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 18 Nov 2024 22:32:22 +0100 Subject: [PATCH 075/230] create_empty_image --- tools/retro-library/download_art.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 9240ae5f3..b7ba14e7f 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -46,6 +46,7 @@ def fetch_image_data(game): data = response.json() img_url = data.get('img') # Usar get para evitar errores si 'img' no existe o es None if img_url: + create_empty_image(name, platform, save_folder) download_image(name, platform, img_url, save_folder) else: print(f"No se encontró una URL de imagen válida para {platform}/{name}. Creando archivo vacío.") From 9065acaf99f50dd4faa5ead0a64dcbf2cbf2b765 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 19 Nov 2024 19:18:34 +0100 Subject: [PATCH 076/230] generateGameLists_retroAchievements --- functions/ToolScripts/emuDeckPlugins.sh | 3 + functions/generateGameLists.sh | 5 ++ tools/retro-library/retro_achievements.py | 71 +++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 tools/retro-library/retro_achievements.py diff --git a/functions/ToolScripts/emuDeckPlugins.sh b/functions/ToolScripts/emuDeckPlugins.sh index 1ca376d8f..f2e54714d 100644 --- a/functions/ToolScripts/emuDeckPlugins.sh +++ b/functions/ToolScripts/emuDeckPlugins.sh @@ -153,6 +153,9 @@ Plugins_installDeckyRomLibrary(){ echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.zip" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/decky-rom-library.zip" echo $password | sudo -S chown $USER:$USER -R $HOME/homebrew/plugins/decky-rom-library fi + #RAachievemets + rau=$(cat "$HOME/.config/EmuDeck/.rau") + setSetting cheevos_username $rau } diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 980941bac..4514f6f8e 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -113,4 +113,9 @@ generateGameLists_extraArtwork() { json=$(jq -n --arg grid "$dest_path" '{grid: $grid}') echo "$json" +} + +generateGameLists_retroAchievements(){ + local hash=$1 + python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" } \ No newline at end of file diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py new file mode 100644 index 000000000..a0ae4a1c3 --- /dev/null +++ b/tools/retro-library/retro_achievements.py @@ -0,0 +1,71 @@ +import requests +import sys + +# Credenciales y usuario +API_USERNAME = "dragoonDorise" +API_KEY = "mvLqoKB3JmbXrezCd7LIXzMnV42ApWzj" +USER = sys.argv[1] +md5_to_find = sys.argv[2] + +# Endpoints +BASE_URL = "https://retroachievements.org/API/" +GAMES_LIST_ENDPOINT = f"{BASE_URL}API_GetGameList.php" +GAME_INFO_ENDPOINT = f"{BASE_URL}API_GetGameInfoAndUserProgress.php" + +# Función para obtener la lista de juegos del sistema +def get_games(system_id=1): + response = requests.get(GAMES_LIST_ENDPOINT, params={ + "i": system_id, + "h": 1, + "f": 1, + "z": API_USERNAME, + "y": API_KEY + }) + if response.status_code == 200: + return response.json() + else: + print(f"Error al obtener los juegos: {response.status_code}") + return None + +# Función para obtener información del juego y progreso del usuario +def get_game_info_and_progress(game_id): + response = requests.get(GAME_INFO_ENDPOINT, params={ + "g": game_id, + "u": USER, + "z": API_USERNAME, + "y": API_KEY + }) + if response.status_code == 200: + return response.json() + else: + print(f"Error al obtener información del juego: {response.status_code}") + return None + +# Función principal +def main(): + # Obtener lista de juegos + games = get_games() + if not games: + return + + # Buscar juego por MD5 + game_id = None + for game in games: + hashes = game.get("Hashes", []) + if md5_to_find in hashes: + game_id = game.get("ID") + print(f"Juego encontrado: {game['Title']} (ID: {game_id})") + break + + if game_id: + # Obtener datos del juego y progreso del usuario + game_data = get_game_info_and_progress(game_id) + if game_data: + print("Datos del juego y progreso del usuario:") + print(game_data) + else: + print("Hash MD5 no encontrado en la lista de juegos.") + +# Ejecutar script +if __name__ == "__main__": + main() From 6d62933b7cb45fb5c83b42a1bd8b2bea1736909f Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 19 Nov 2024 19:20:59 +0100 Subject: [PATCH 077/230] platform id --- tools/retro-library/retro_achievements.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py index a0ae4a1c3..07b94ab8c 100644 --- a/tools/retro-library/retro_achievements.py +++ b/tools/retro-library/retro_achievements.py @@ -6,14 +6,14 @@ API_KEY = "mvLqoKB3JmbXrezCd7LIXzMnV42ApWzj" USER = sys.argv[1] md5_to_find = sys.argv[2] - +SYSTEM_ID = sys.argv[3] # Endpoints BASE_URL = "https://retroachievements.org/API/" GAMES_LIST_ENDPOINT = f"{BASE_URL}API_GetGameList.php" GAME_INFO_ENDPOINT = f"{BASE_URL}API_GetGameInfoAndUserProgress.php" # Función para obtener la lista de juegos del sistema -def get_games(system_id=1): +def get_games(system_id): response = requests.get(GAMES_LIST_ENDPOINT, params={ "i": system_id, "h": 1, @@ -44,7 +44,7 @@ def get_game_info_and_progress(game_id): # Función principal def main(): # Obtener lista de juegos - games = get_games() + games = get_games(SYSTEM_ID) if not games: return From 74c8f71b6f4864b5652f05c8ca51f2aeb15db2e3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 19 Nov 2024 19:24:51 +0100 Subject: [PATCH 078/230] ID --- functions/generateGameLists.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 4514f6f8e..22bd1f962 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -117,5 +117,6 @@ generateGameLists_extraArtwork() { generateGameLists_retroAchievements(){ local hash=$1 - python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" + local systemID=$2 + python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" "$systemID" } \ No newline at end of file From 6e9308c9df36b30e0563cc306cc8d002fb5201e8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 19 Nov 2024 19:28:12 +0100 Subject: [PATCH 079/230] ignore warning --- tools/retro-library/retro_achievements.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py index 07b94ab8c..ee5a89f1c 100644 --- a/tools/retro-library/retro_achievements.py +++ b/tools/retro-library/retro_achievements.py @@ -1,5 +1,9 @@ import requests import sys +import warnings +from requests.exceptions import RequestsDependencyWarning + +warnings.filterwarnings("ignore", category=RequestsDependencyWarning) # Credenciales y usuario API_USERNAME = "dragoonDorise" @@ -54,14 +58,12 @@ def main(): hashes = game.get("Hashes", []) if md5_to_find in hashes: game_id = game.get("ID") - print(f"Juego encontrado: {game['Title']} (ID: {game_id})") break if game_id: # Obtener datos del juego y progreso del usuario game_data = get_game_info_and_progress(game_id) if game_data: - print("Datos del juego y progreso del usuario:") print(game_data) else: print("Hash MD5 no encontrado en la lista de juegos.") From 1739ca9b1d4801493b992681253c0cc62d899832 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 19 Nov 2024 19:29:53 +0100 Subject: [PATCH 080/230] charset --- .../charset_normalizer/.codecov.yml | 8 + .../charset_normalizer/.coveragerc | 2 + .../charset_normalizer/.github/FUNDING.yml | 4 + .../.github/ISSUE_TEMPLATE/bug_report.md | 28 + .../.github/ISSUE_TEMPLATE/feature_request.md | 20 + .../.github/ISSUE_TEMPLATE/wrong_charset.md | 39 + .../charset_normalizer/.github/dependabot.yml | 20 + .../.github/workflows/cd.yml | 160 ++ .../.github/workflows/ci.yml | 252 +++ .../.github/workflows/codeql.yml | 57 + .../.github/workflows/scorecards.yml | 71 + .../charset_normalizer/.gitignore | 127 ++ .../.pre-commit-config.yaml | 31 + .../charset_normalizer/.readthedocs.yaml | 19 + .../charset_normalizer/CHANGELOG.md | 393 ++++ .../charset_normalizer/CODE_OF_CONDUCT.md | 76 + .../charset_normalizer/CONTRIBUTING.md | 72 + .../retro-library/charset_normalizer/LICENSE | 21 + .../charset_normalizer/MANIFEST.in | 5 + .../charset_normalizer/README.md | 257 +++ .../charset_normalizer/SECURITY.md | 4 + .../charset_normalizer/UPGRADE.md | 31 + .../charset_normalizer/bin/bc.py | 109 + .../charset_normalizer/bin/coverage.py | 106 + .../charset_normalizer/bin/integration.py | 56 + .../charset_normalizer/bin/performance.py | 155 ++ .../charset_normalizer/bin/serve.py | 51 + .../charset_normalizer/__init__.py | 47 + .../charset_normalizer/__main__.py | 6 + .../charset_normalizer/api.py | 670 ++++++ .../charset_normalizer/cd.py | 395 ++++ .../charset_normalizer/cli/__init__.py | 8 + .../charset_normalizer/cli/__main__.py | 321 +++ .../charset_normalizer/constant.py | 1998 +++++++++++++++++ .../charset_normalizer/legacy.py | 65 + .../charset_normalizer/md.py | 629 ++++++ .../charset_normalizer/models.py | 359 +++ .../charset_normalizer/py.typed | 0 .../charset_normalizer/utils.py | 421 ++++ .../charset_normalizer/version.py | 8 + .../charset_normalizer/data/NOTICE.md | 9 + .../data/sample-arabic-1.txt | 6 + .../charset_normalizer/data/sample-arabic.txt | 6 + .../data/sample-bulgarian.txt | 7 + .../data/sample-chinese.txt | 14 + .../data/sample-english.bom.txt | 35 + .../data/sample-french-1.txt | 59 + .../charset_normalizer/data/sample-french.txt | 59 + .../data/sample-greek-2.txt | 1 + .../charset_normalizer/data/sample-greek.txt | 1 + .../data/sample-hebrew-2.txt | 1 + .../data/sample-hebrew-3.txt | 1 + .../charset_normalizer/data/sample-korean.txt | 1 + .../charset_normalizer/data/sample-polish.txt | 204 ++ .../data/sample-russian-2.txt | 5 + .../data/sample-russian-3.txt | 7 + .../data/sample-russian.txt | 5 + .../data/sample-spanish.txt | 33 + .../data/sample-turkish.txt | 33 + .../charset_normalizer/dev-requirements.txt | 7 + .../charset_normalizer/docs/Makefile | 20 + .../charset_normalizer/docs/api.rst | 100 + .../charset_normalizer/docs/community/faq.rst | 75 + .../docs/community/featured.rst | 51 + .../docs/community/speedup.rst | 50 + .../docs/community/why_migrate.rst | 18 + .../charset_normalizer/docs/conf.py | 170 ++ .../charset_normalizer/docs/index.rst | 93 + .../charset_normalizer/docs/make.bat | 36 + .../charset_normalizer/docs/requirements.txt | 2 + .../docs/user/advanced_search.rst | 82 + .../charset_normalizer/docs/user/cli.rst | 116 + .../docs/user/getstarted.rst | 73 + .../docs/user/handling_result.rst | 25 + .../docs/user/miscellaneous.rst | 64 + .../charset_normalizer/docs/user/support.rst | 183 ++ .../charset_normalizer/pyproject.toml | 81 + .../charset_normalizer/setup.cfg | 3 + .../retro-library/charset_normalizer/setup.py | 30 + .../charset_normalizer/tests/__init__.py | 0 .../tests/test_base_detection.py | 162 ++ .../charset_normalizer/tests/test_cli.py | 116 + .../tests/test_coherence_detection.py | 108 + .../tests/test_detect_legacy.py | 43 + .../tests/test_edge_case.py | 59 + .../tests/test_full_detection.py | 50 + .../charset_normalizer/tests/test_isbinary.py | 29 + .../tests/test_large_payload.py | 55 + .../charset_normalizer/tests/test_logging.py | 53 + .../tests/test_mess_detection.py | 48 + .../tests/test_preemptive_detection.py | 92 + .../charset_normalizer/tests/test_utils.py | 52 + 92 files changed, 9734 insertions(+) create mode 100644 tools/retro-library/charset_normalizer/.codecov.yml create mode 100644 tools/retro-library/charset_normalizer/.coveragerc create mode 100644 tools/retro-library/charset_normalizer/.github/FUNDING.yml create mode 100644 tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md create mode 100644 tools/retro-library/charset_normalizer/.github/dependabot.yml create mode 100644 tools/retro-library/charset_normalizer/.github/workflows/cd.yml create mode 100644 tools/retro-library/charset_normalizer/.github/workflows/ci.yml create mode 100644 tools/retro-library/charset_normalizer/.github/workflows/codeql.yml create mode 100644 tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml create mode 100644 tools/retro-library/charset_normalizer/.gitignore create mode 100644 tools/retro-library/charset_normalizer/.pre-commit-config.yaml create mode 100644 tools/retro-library/charset_normalizer/.readthedocs.yaml create mode 100644 tools/retro-library/charset_normalizer/CHANGELOG.md create mode 100644 tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md create mode 100644 tools/retro-library/charset_normalizer/CONTRIBUTING.md create mode 100644 tools/retro-library/charset_normalizer/LICENSE create mode 100644 tools/retro-library/charset_normalizer/MANIFEST.in create mode 100644 tools/retro-library/charset_normalizer/README.md create mode 100644 tools/retro-library/charset_normalizer/SECURITY.md create mode 100644 tools/retro-library/charset_normalizer/UPGRADE.md create mode 100644 tools/retro-library/charset_normalizer/bin/bc.py create mode 100644 tools/retro-library/charset_normalizer/bin/coverage.py create mode 100644 tools/retro-library/charset_normalizer/bin/integration.py create mode 100644 tools/retro-library/charset_normalizer/bin/performance.py create mode 100644 tools/retro-library/charset_normalizer/bin/serve.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/__init__.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/__main__.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/api.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/cd.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/cli/__init__.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/cli/__main__.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/constant.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/legacy.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/md.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/models.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/py.typed create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/utils.py create mode 100644 tools/retro-library/charset_normalizer/charset_normalizer/version.py create mode 100644 tools/retro-library/charset_normalizer/data/NOTICE.md create mode 100644 tools/retro-library/charset_normalizer/data/sample-arabic-1.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-arabic.txt create mode 100755 tools/retro-library/charset_normalizer/data/sample-bulgarian.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-chinese.txt create mode 100755 tools/retro-library/charset_normalizer/data/sample-english.bom.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-french-1.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-french.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-greek-2.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-greek.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt create mode 100755 tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-korean.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-polish.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-russian-2.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-russian-3.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-russian.txt create mode 100755 tools/retro-library/charset_normalizer/data/sample-spanish.txt create mode 100644 tools/retro-library/charset_normalizer/data/sample-turkish.txt create mode 100644 tools/retro-library/charset_normalizer/dev-requirements.txt create mode 100755 tools/retro-library/charset_normalizer/docs/Makefile create mode 100644 tools/retro-library/charset_normalizer/docs/api.rst create mode 100644 tools/retro-library/charset_normalizer/docs/community/faq.rst create mode 100644 tools/retro-library/charset_normalizer/docs/community/featured.rst create mode 100644 tools/retro-library/charset_normalizer/docs/community/speedup.rst create mode 100644 tools/retro-library/charset_normalizer/docs/community/why_migrate.rst create mode 100755 tools/retro-library/charset_normalizer/docs/conf.py create mode 100755 tools/retro-library/charset_normalizer/docs/index.rst create mode 100644 tools/retro-library/charset_normalizer/docs/make.bat create mode 100755 tools/retro-library/charset_normalizer/docs/requirements.txt create mode 100644 tools/retro-library/charset_normalizer/docs/user/advanced_search.rst create mode 100644 tools/retro-library/charset_normalizer/docs/user/cli.rst create mode 100644 tools/retro-library/charset_normalizer/docs/user/getstarted.rst create mode 100644 tools/retro-library/charset_normalizer/docs/user/handling_result.rst create mode 100644 tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst create mode 100644 tools/retro-library/charset_normalizer/docs/user/support.rst create mode 100644 tools/retro-library/charset_normalizer/pyproject.toml create mode 100644 tools/retro-library/charset_normalizer/setup.cfg create mode 100644 tools/retro-library/charset_normalizer/setup.py create mode 100644 tools/retro-library/charset_normalizer/tests/__init__.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_base_detection.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_cli.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_coherence_detection.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_detect_legacy.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_edge_case.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_full_detection.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_isbinary.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_large_payload.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_logging.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_mess_detection.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py create mode 100644 tools/retro-library/charset_normalizer/tests/test_utils.py diff --git a/tools/retro-library/charset_normalizer/.codecov.yml b/tools/retro-library/charset_normalizer/.codecov.yml new file mode 100644 index 000000000..f307895e7 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + target: 88% + threshold: null + patch: false + changes: false diff --git a/tools/retro-library/charset_normalizer/.coveragerc b/tools/retro-library/charset_normalizer/.coveragerc new file mode 100644 index 000000000..723bfd0cb --- /dev/null +++ b/tools/retro-library/charset_normalizer/.coveragerc @@ -0,0 +1,2 @@ +[run] +source=charset_normalizer diff --git a/tools/retro-library/charset_normalizer/.github/FUNDING.yml b/tools/retro-library/charset_normalizer/.github/FUNDING.yml new file mode 100644 index 000000000..e793ecd50 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms +tidelift: pypi/charset-normalizer +github: + - Ousret diff --git a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..a8be5653b --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Create a report to help us fix something bad like an exception +title: "[BUG]" +labels: bug, help wanted +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug/exception is. + +**To Reproduce** +Give us the target text file. Host it somewhere with untouched encoding. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Logs** +If applicable, add console outputs to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. Linux, Windows or Mac] + - Python version [e.g. 3.5] + - Package version [eg. 2.0.0] + +**Additional context** +Add any other context about the problem here. diff --git a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..44d549f13 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[Proposal]" +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md new file mode 100644 index 000000000..e90d866d3 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md @@ -0,0 +1,39 @@ +--- +name: Wrong charset / Detection issue +about: Create a report to help us improve the detection mechanism +title: "[DETECTION]" +labels: help wanted, detection +assignees: '' + +--- + +**Notice** +I hereby announce that my raw input is not : +- Too small content (<=32 characters) as I do know that ANY charset detector heavily depends on content +- Encoded in a deprecated/abandoned encoding that is not even supported by my interpreter + +**Provide the file** +A accessible way of retrieving the file concerned. Host it somewhere with untouched encoding. + +**Verbose output** +Using the CLI, run `normalizer -v ./my-file.txt` and past the result in here. + +``` +(venv) >normalizer -v ./data/sample.1.ar.srt +2021-05-21 08:38:44,050 | DEBUG | ascii does not fit given bytes sequence at ALL. 'ascii' codec can't decode byte 0xca in position 54: ordinal not in range(128) +2021-05-21 08:38:44,051 | DEBUG | big5 does not fit given bytes sequence at ALL. 'big5' codec can't decode byte 0xc9 in position 60: illegal multibyte sequence +2021-05-21 08:38:44,051 | DEBUG | big5hkscs does not fit given bytes sequence at ALL. 'big5hkscs' codec can't decode byte 0xc9 in position 60: illegal multibyte sequence +.... +``` + +**Expected encoding** +A clear and concise description of what you expected as encoding. Any more details about how the current guess is wrong +is very much appreciated. + +**Desktop (please complete the following information):** + - OS: [e.g. Linux, Windows or Mac] + - Python version [e.g. 3.5] + - Package version [eg. 2.0.0] + +**Additional context** +Add any other context about the problem here. diff --git a/tools/retro-library/charset_normalizer/.github/dependabot.yml b/tools/retro-library/charset_normalizer/.github/dependabot.yml new file mode 100644 index 000000000..d01328580 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/dependabot.yml @@ -0,0 +1,20 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: pip + directory: /docs + schedule: + interval: daily diff --git a/tools/retro-library/charset_normalizer/.github/workflows/cd.yml b/tools/retro-library/charset_normalizer/.github/workflows/cd.yml new file mode 100644 index 000000000..75ef10a66 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/workflows/cd.yml @@ -0,0 +1,160 @@ +name: Continuous Delivery + +on: + workflow_dispatch: + + release: + types: + - created + +permissions: + contents: read + +jobs: + pre_flight_check: + name: Preflight Checks + uses: ./.github/workflows/ci.yml + + universal-wheel: + name: Build Universal Wheel + runs-on: ubuntu-latest + needs: + - pre_flight_check + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.11' + - name: Update pip, install build + run: | + python -m pip install --upgrade pip + python -m pip install build + - name: Build Wheel + env: + CHARSET_NORMALIZER_USE_MYPYC: '0' + run: python -m build + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: dist + path: dist + + build-wheels: + name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }} + runs-on: ${{ matrix.os }} + needs: pre_flight_check + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-13 ] + qemu: [ '' ] + include: + # Split ubuntu job for the sake of speed-up + - os: ubuntu-latest + qemu: aarch64 + - os: ubuntu-latest + qemu: ppc64le + - os: ubuntu-latest + qemu: s390x + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + - name: Set up QEMU + if: ${{ matrix.qemu }} + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + with: + platforms: all + id: qemu + - name: Prepare emulation + run: | + if [[ -n "${{ matrix.qemu }}" ]]; then + # Build emulated architectures only if QEMU is set, + # use default "auto" otherwise + echo "CIBW_ARCHS_LINUX=${{ matrix.qemu }}" >> $GITHUB_ENV + fi + shell: bash + - name: Setup Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + - name: Update pip, wheel, setuptools, build, twine + run: | + python -m pip install -U pip wheel setuptools build twine + - name: Build wheels + uses: pypa/cibuildwheel@7940a4c0e76eb2030e473a5f864f291f63ee879b # v2.21.3 + env: + CIBW_BUILD_FRONTEND: build + CIBW_ARCHS_MACOS: x86_64 arm64 universal2 + CIBW_ENVIRONMENT: CHARSET_NORMALIZER_USE_MYPYC='1' + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest -c {package} {package}/tests + CIBW_SKIP: pp* cp36* + - name: Upload artifacts + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: dist + path: ./wheelhouse/*.whl + + checksum: + name: Compute hashes + runs-on: ubuntu-latest + needs: + - build-wheels + - universal-wheel + outputs: + hashes: ${{ steps.compute.outputs.hashes }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Download distributions + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a + with: + name: dist + path: dist + - name: Collected dists + run: | + tree dist + - name: Generate hashes + id: compute # needs.checksum.outputs.hashes + working-directory: ./dist + run: echo "hashes=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT + + provenance: + needs: checksum + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 + permissions: + actions: read + id-token: write + contents: write + with: + base64-subjects: ${{ needs.checksum.outputs.hashes }} + upload-assets: true + compile-generator: true + + deploy: + name: 🚀 Deploy to PyPi + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + permissions: + id-token: write + contents: write + needs: provenance + environment: + name: pypi + url: https://pypi.org/project/charset-normalizer/ + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Download distributions + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a + with: + name: dist + path: dist + - name: Collected dists + run: | + tree dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # release/v1 + - name: Upload dists to GitHub Release + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + run: | + gh release upload ${{ github.ref_name }} dist/* --repo ${{ github.repository }} diff --git a/tools/retro-library/charset_normalizer/.github/workflows/ci.yml b/tools/retro-library/charset_normalizer/.github/workflows/ci.yml new file mode 100644 index 000000000..23433aab7 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/workflows/ci.yml @@ -0,0 +1,252 @@ +name: Continuous Integration + +on: + workflow_call: + pull_request: + push: + branches: + - master + +permissions: + contents: read + +jobs: + lint: + name: 🎨 Linters + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install -U pip setuptools + python -m pip install -r dev-requirements.txt + python -m pip uninstall -y charset-normalizer + - name: Pre-commit checks + run: | + pre-commit run --all + + tests: + name: ✅ Tests + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install dependencies + run: | + python -m pip install -U pip setuptools + python -m pip install -r dev-requirements.txt + python -m pip uninstall -y charset-normalizer + - name: Install the package + run: | + python -m build + python -m pip install ./dist/*.whl + - name: Run tests + run: | + pytest + - uses: codecov/codecov-action@4fe8c5f003fae66aa5ebb77cfd3e7bfbbda0b6b0 # v3.1.5 + + detection_coverage: + + needs: + - tests + + name: 📈 Detection Coverage + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install -U pip setuptools + python -m pip install -r dev-requirements.txt + python -m pip uninstall -y charset-normalizer + - name: Install the package + run: | + python -m build + python -m pip install ./dist/*.whl + - name: Clone the complete dataset + run: | + git clone https://github.com/Ousret/char-dataset.git + - name: Coverage WITH preemptive + run: | + python ./bin/coverage.py --coverage 97 --with-preemptive + - name: Coverage WITHOUT preemptive + run: | + python ./bin/coverage.py --coverage 95 + +# integration_test: +# +# needs: +# - tests +# +# name: 🔗 Integration Tests +# runs-on: ubuntu-latest +# +# steps: +# - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 +# - name: Set up Python +# uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 +# with: +# python-version: '3.11' +# - name: Install dependencies +# run: | +# pip install -U pip setuptools +# pip install -r dev-requirements.txt +# - name: Remove Chardet & Charset-Normalizer +# run: | +# pip uninstall -y chardet +# pip uninstall -y charset-normalizer +# - name: Install the package +# run: | +# python -m build +# pip install ./dist/*.whl +# - name: Clone the complete dataset +# run: | +# git clone https://github.com/Ousret/char-dataset.git +# - name: Start the Flask server +# run: | +# python ./bin/serve.py & +# - name: Integration Tests with Requests +# run: | +# python ./bin/integration.py + + chardet_bc: + + name: ⏪ Chardet Backward-Compatibility Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install -U pip setuptools + python -m pip install -r dev-requirements.txt + python -m pip uninstall -y charset-normalizer + - name: Install the package + run: | + python -m build + python -m pip install ./dist/*.whl + - name: Clone the complete dataset + run: | + git clone https://github.com/Ousret/char-dataset.git + - name: BC Coverage + run: | + python ./bin/bc.py --coverage 80 + + mypyc_test: + + name: ⚡ MypyC Tests + + needs: + - tests + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + os: [ ubuntu-latest, macos-latest, windows-latest ] + include: + - python-version: "3.7" + os: ubuntu-latest + - python-version: "3.7" + os: macos-13 + - python-version: "3.7" + os: windows-latest + env: + PYTHONIOENCODING: utf8 # only needed for Windows (console IO output encoding) + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install dependencies + run: | + python -m pip install -U pip setuptools + python -m pip install -r dev-requirements.txt + python -m pip uninstall -y charset-normalizer + - name: Install the package + env: + CHARSET_NORMALIZER_USE_MYPYC: '1' + run: | + python -m pip install . + - name: Clone the complete dataset + run: | + git clone https://github.com/Ousret/char-dataset.git + - name: Coverage WITH preemptive + run: | + python ./bin/coverage.py --coverage 97 --with-preemptive + - name: Performance (Normal) + run: | + python ./bin/performance.py + + performance: + name: ⚡ Performance Test (no MypyC) + runs-on: ubuntu-latest + + needs: + - mypyc_test + - chardet_bc + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.11' + - name: Install dependencies + run: | + python -m pip install -U pip setuptools + python -m pip install -r dev-requirements.txt + python -m pip uninstall -y charset-normalizer + - name: Install the package + run: | + python -m build + python -m pip install ./dist/*.whl + - name: Clone the complete dataset + run: | + git clone https://github.com/Ousret/char-dataset.git + - name: Performance (Normal) + run: | + python ./bin/performance.py + - name: Performance (Medium) + run: | + python ./bin/performance.py --size-increase 2 diff --git a/tools/retro-library/charset_normalizer/.github/workflows/codeql.yml b/tools/retro-library/charset_normalizer/.github/workflows/codeql.yml new file mode 100644 index 000000000..fee750870 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/workflows/codeql.yml @@ -0,0 +1,57 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +permissions: + contents: read + +on: + push: + branches: [ "master", "2.1.x" ] + pull_request: + branches: [ "master", "2.1.x" ] + schedule: + - cron: '39 1 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 + with: + languages: ${{ matrix.language }} + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 + with: + category: "/language:${{matrix.language}}" diff --git a/tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml b/tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml new file mode 100644 index 000000000..2dbbaeff2 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml @@ -0,0 +1,71 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '20 7 * * 2' + push: + branches: ["master"] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + contents: read + actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecards on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 + with: + sarif_file: results.sarif diff --git a/tools/retro-library/charset_normalizer/.gitignore b/tools/retro-library/charset_normalizer/.gitignore new file mode 100644 index 000000000..d22c75f63 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.gitignore @@ -0,0 +1,127 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +.idea/ +char-dataset/ diff --git a/tools/retro-library/charset_normalizer/.pre-commit-config.yaml b/tools/retro-library/charset_normalizer/.pre-commit-config.yaml new file mode 100644 index 000000000..09fa625e5 --- /dev/null +++ b/tools/retro-library/charset_normalizer/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +exclude: 'docs/|data/|tests/' + +repos: + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: ["--py37-plus"] + + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + args: ["--target-version", "py37"] + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + additional_dependencies: [flake8-2020] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.13.0 + hooks: + - id: mypy + exclude: 'tests/|bin/' diff --git a/tools/retro-library/charset_normalizer/.readthedocs.yaml b/tools/retro-library/charset_normalizer/.readthedocs.yaml new file mode 100644 index 000000000..b783a32fb --- /dev/null +++ b/tools/retro-library/charset_normalizer/.readthedocs.yaml @@ -0,0 +1,19 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +# formats: +# - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt diff --git a/tools/retro-library/charset_normalizer/CHANGELOG.md b/tools/retro-library/charset_normalizer/CHANGELOG.md new file mode 100644 index 000000000..608567e41 --- /dev/null +++ b/tools/retro-library/charset_normalizer/CHANGELOG.md @@ -0,0 +1,393 @@ +# Changelog +All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [3.4.1](https://github.com/Ousret/charset_normalizer/compare/3.4.0...master) (2024-10-??) + +### Changed +- Project metadata are now stored using `pyproject.toml` instead of `setup.cfg` using setuptools as the build backend. +- Enforce annotation delayed loading for a simpler and consistent types in the project. + +### Added +- pre-commit configuration. + +### Removed +- `build-requirements.txt` as per using `pyproject.toml` native build configuration. + +## [3.4.0](https://github.com/Ousret/charset_normalizer/compare/3.3.2...3.4.0) (2024-10-08) + +### Added +- Argument `--no-preemptive` in the CLI to prevent the detector to search for hints. +- Support for Python 3.13 (#512) + +### Fixed +- Relax the TypeError exception thrown when trying to compare a CharsetMatch with anything else than a CharsetMatch. +- Improved the general reliability of the detector based on user feedbacks. (#520) (#509) (#498) (#407) (#537) +- Declared charset in content (preemptive detection) not changed when converting to utf-8 bytes. (#381) + +## [3.3.2](https://github.com/Ousret/charset_normalizer/compare/3.3.1...3.3.2) (2023-10-31) + +### Fixed +- Unintentional memory usage regression when using large payload that match several encoding (#376) +- Regression on some detection case showcased in the documentation (#371) + +### Added +- Noise (md) probe that identify malformed arabic representation due to the presence of letters in isolated form (credit to my wife) + +## [3.3.1](https://github.com/Ousret/charset_normalizer/compare/3.3.0...3.3.1) (2023-10-22) + +### Changed +- Optional mypyc compilation upgraded to version 1.6.1 for Python >= 3.8 +- Improved the general detection reliability based on reports from the community + +## [3.3.0](https://github.com/Ousret/charset_normalizer/compare/3.2.0...3.3.0) (2023-09-30) + +### Added +- Allow to execute the CLI (e.g. normalizer) through `python -m charset_normalizer.cli` or `python -m charset_normalizer` +- Support for 9 forgotten encoding that are supported by Python but unlisted in `encoding.aliases` as they have no alias (#323) + +### Removed +- (internal) Redundant utils.is_ascii function and unused function is_private_use_only +- (internal) charset_normalizer.assets is moved inside charset_normalizer.constant + +### Changed +- (internal) Unicode code blocks in constants are updated using the latest v15.0.0 definition to improve detection +- Optional mypyc compilation upgraded to version 1.5.1 for Python >= 3.8 + +### Fixed +- Unable to properly sort CharsetMatch when both chaos/noise and coherence were close due to an unreachable condition in \_\_lt\_\_ (#350) + +## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07) + +### Changed +- Typehint for function `from_path` no longer enforce `PathLike` as its first argument +- Minor improvement over the global detection reliability + +### Added +- Introduce function `is_binary` that relies on main capabilities, and optimized to detect binaries +- Propagate `enable_fallback` argument throughout `from_bytes`, `from_path`, and `from_fp` that allow a deeper control over the detection (default True) +- Explicit support for Python 3.12 + +### Fixed +- Edge case detection failure where a file would contain 'very-long' camel cased word (Issue #289) + +## [3.1.0](https://github.com/Ousret/charset_normalizer/compare/3.0.1...3.1.0) (2023-03-06) + +### Added +- Argument `should_rename_legacy` for legacy function `detect` and disregard any new arguments without errors (PR #262) + +### Removed +- Support for Python 3.6 (PR #260) + +### Changed +- Optional speedup provided by mypy/c 1.0.1 + +## [3.0.1](https://github.com/Ousret/charset_normalizer/compare/3.0.0...3.0.1) (2022-11-18) + +### Fixed +- Multi-bytes cutter/chunk generator did not always cut correctly (PR #233) + +### Changed +- Speedup provided by mypy/c 0.990 on Python >= 3.7 + +## [3.0.0](https://github.com/Ousret/charset_normalizer/compare/2.1.1...3.0.0) (2022-10-20) + +### Added +- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results +- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES +- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio +- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl) + +### Changed +- Build with static metadata using 'build' frontend +- Make the language detection stricter +- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1 + +### Fixed +- CLI with opt --normalize fail when using full path for files +- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it +- Sphinx warnings when generating the documentation + +### Removed +- Coherence detector no longer return 'Simple English' instead return 'English' +- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese' +- Breaking: Method `first()` and `best()` from CharsetMatch +- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII) +- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches +- Breaking: Top-level function `normalize` +- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch +- Support for the backport `unicodedata2` + +## [3.0.0rc1](https://github.com/Ousret/charset_normalizer/compare/3.0.0b2...3.0.0rc1) (2022-10-18) + +### Added +- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results +- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES +- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio + +### Changed +- Build with static metadata using 'build' frontend +- Make the language detection stricter + +### Fixed +- CLI with opt --normalize fail when using full path for files +- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it + +### Removed +- Coherence detector no longer return 'Simple English' instead return 'English' +- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese' + +## [3.0.0b2](https://github.com/Ousret/charset_normalizer/compare/3.0.0b1...3.0.0b2) (2022-08-21) + +### Added +- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl) + +### Removed +- Breaking: Method `first()` and `best()` from CharsetMatch +- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII) + +### Fixed +- Sphinx warnings when generating the documentation + +## [3.0.0b1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...3.0.0b1) (2022-08-15) + +### Changed +- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1 + +### Removed +- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches +- Breaking: Top-level function `normalize` +- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch +- Support for the backport `unicodedata2` + +## [2.1.1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...2.1.1) (2022-08-19) + +### Deprecated +- Function `normalize` scheduled for removal in 3.0 + +### Changed +- Removed useless call to decode in fn is_unprintable (#206) + +### Fixed +- Third-party library (i18n xgettext) crashing not recognizing utf_8 (PEP 263) with underscore from [@aleksandernovikov](https://github.com/aleksandernovikov) (#204) + +## [2.1.0](https://github.com/Ousret/charset_normalizer/compare/2.0.12...2.1.0) (2022-06-19) + +### Added +- Output the Unicode table version when running the CLI with `--version` (PR #194) + +### Changed +- Re-use decoded buffer for single byte character sets from [@nijel](https://github.com/nijel) (PR #175) +- Fixing some performance bottlenecks from [@deedy5](https://github.com/deedy5) (PR #183) + +### Fixed +- Workaround potential bug in cpython with Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space (PR #175) +- CLI default threshold aligned with the API threshold from [@oleksandr-kuzmenko](https://github.com/oleksandr-kuzmenko) (PR #181) + +### Removed +- Support for Python 3.5 (PR #192) + +### Deprecated +- Use of backport unicodedata from `unicodedata2` as Python is quickly catching up, scheduled for removal in 3.0 (PR #194) + +## [2.0.12](https://github.com/Ousret/charset_normalizer/compare/2.0.11...2.0.12) (2022-02-12) + +### Fixed +- ASCII miss-detection on rare cases (PR #170) + +## [2.0.11](https://github.com/Ousret/charset_normalizer/compare/2.0.10...2.0.11) (2022-01-30) + +### Added +- Explicit support for Python 3.11 (PR #164) + +### Changed +- The logging behavior have been completely reviewed, now using only TRACE and DEBUG levels (PR #163 #165) + +## [2.0.10](https://github.com/Ousret/charset_normalizer/compare/2.0.9...2.0.10) (2022-01-04) + +### Fixed +- Fallback match entries might lead to UnicodeDecodeError for large bytes sequence (PR #154) + +### Changed +- Skipping the language-detection (CD) on ASCII (PR #155) + +## [2.0.9](https://github.com/Ousret/charset_normalizer/compare/2.0.8...2.0.9) (2021-12-03) + +### Changed +- Moderating the logging impact (since 2.0.8) for specific environments (PR #147) + +### Fixed +- Wrong logging level applied when setting kwarg `explain` to True (PR #146) + +## [2.0.8](https://github.com/Ousret/charset_normalizer/compare/2.0.7...2.0.8) (2021-11-24) +### Changed +- Improvement over Vietnamese detection (PR #126) +- MD improvement on trailing data and long foreign (non-pure latin) data (PR #124) +- Efficiency improvements in cd/alphabet_languages from [@adbar](https://github.com/adbar) (PR #122) +- call sum() without an intermediary list following PEP 289 recommendations from [@adbar](https://github.com/adbar) (PR #129) +- Code style as refactored by Sourcery-AI (PR #131) +- Minor adjustment on the MD around european words (PR #133) +- Remove and replace SRTs from assets / tests (PR #139) +- Initialize the library logger with a `NullHandler` by default from [@nmaynes](https://github.com/nmaynes) (PR #135) +- Setting kwarg `explain` to True will add provisionally (bounded to function lifespan) a specific stream handler (PR #135) + +### Fixed +- Fix large (misleading) sequence giving UnicodeDecodeError (PR #137) +- Avoid using too insignificant chunk (PR #137) + +### Added +- Add and expose function `set_logging_handler` to configure a specific StreamHandler from [@nmaynes](https://github.com/nmaynes) (PR #135) +- Add `CHANGELOG.md` entries, format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) (PR #141) + +## [2.0.7](https://github.com/Ousret/charset_normalizer/compare/2.0.6...2.0.7) (2021-10-11) +### Added +- Add support for Kazakh (Cyrillic) language detection (PR #109) + +### Changed +- Further, improve inferring the language from a given single-byte code page (PR #112) +- Vainly trying to leverage PEP263 when PEP3120 is not supported (PR #116) +- Refactoring for potential performance improvements in loops from [@adbar](https://github.com/adbar) (PR #113) +- Various detection improvement (MD+CD) (PR #117) + +### Removed +- Remove redundant logging entry about detected language(s) (PR #115) + +### Fixed +- Fix a minor inconsistency between Python 3.5 and other versions regarding language detection (PR #117 #102) + +## [2.0.6](https://github.com/Ousret/charset_normalizer/compare/2.0.5...2.0.6) (2021-09-18) +### Fixed +- Unforeseen regression with the loss of the backward-compatibility with some older minor of Python 3.5.x (PR #100) +- Fix CLI crash when using --minimal output in certain cases (PR #103) + +### Changed +- Minor improvement to the detection efficiency (less than 1%) (PR #106 #101) + +## [2.0.5](https://github.com/Ousret/charset_normalizer/compare/2.0.4...2.0.5) (2021-09-14) +### Changed +- The project now comply with: flake8, mypy, isort and black to ensure a better overall quality (PR #81) +- The BC-support with v1.x was improved, the old staticmethods are restored (PR #82) +- The Unicode detection is slightly improved (PR #93) +- Add syntax sugar \_\_bool\_\_ for results CharsetMatches list-container (PR #91) + +### Removed +- The project no longer raise warning on tiny content given for detection, will be simply logged as warning instead (PR #92) + +### Fixed +- In some rare case, the chunks extractor could cut in the middle of a multi-byte character and could mislead the mess detection (PR #95) +- Some rare 'space' characters could trip up the UnprintablePlugin/Mess detection (PR #96) +- The MANIFEST.in was not exhaustive (PR #78) + +## [2.0.4](https://github.com/Ousret/charset_normalizer/compare/2.0.3...2.0.4) (2021-07-30) +### Fixed +- The CLI no longer raise an unexpected exception when no encoding has been found (PR #70) +- Fix accessing the 'alphabets' property when the payload contains surrogate characters (PR #68) +- The logger could mislead (explain=True) on detected languages and the impact of one MBCS match (PR #72) +- Submatch factoring could be wrong in rare edge cases (PR #72) +- Multiple files given to the CLI were ignored when publishing results to STDOUT. (After the first path) (PR #72) +- Fix line endings from CRLF to LF for certain project files (PR #67) + +### Changed +- Adjust the MD to lower the sensitivity, thus improving the global detection reliability (PR #69 #76) +- Allow fallback on specified encoding if any (PR #71) + +## [2.0.3](https://github.com/Ousret/charset_normalizer/compare/2.0.2...2.0.3) (2021-07-16) +### Changed +- Part of the detection mechanism has been improved to be less sensitive, resulting in more accurate detection results. Especially ASCII. (PR #63) +- According to the community wishes, the detection will fall back on ASCII or UTF-8 in a last-resort case. (PR #64) + +## [2.0.2](https://github.com/Ousret/charset_normalizer/compare/2.0.1...2.0.2) (2021-07-15) +### Fixed +- Empty/Too small JSON payload miss-detection fixed. Report from [@tseaver](https://github.com/tseaver) (PR #59) + +### Changed +- Don't inject unicodedata2 into sys.modules from [@akx](https://github.com/akx) (PR #57) + +## [2.0.1](https://github.com/Ousret/charset_normalizer/compare/2.0.0...2.0.1) (2021-07-13) +### Fixed +- Make it work where there isn't a filesystem available, dropping assets frequencies.json. Report from [@sethmlarson](https://github.com/sethmlarson). (PR #55) +- Using explain=False permanently disable the verbose output in the current runtime (PR #47) +- One log entry (language target preemptive) was not show in logs when using explain=True (PR #47) +- Fix undesired exception (ValueError) on getitem of instance CharsetMatches (PR #52) + +### Changed +- Public function normalize default args values were not aligned with from_bytes (PR #53) + +### Added +- You may now use charset aliases in cp_isolation and cp_exclusion arguments (PR #47) + +## [2.0.0](https://github.com/Ousret/charset_normalizer/compare/1.4.1...2.0.0) (2021-07-02) +### Changed +- 4x to 5 times faster than the previous 1.4.0 release. At least 2x faster than Chardet. +- Accent has been made on UTF-8 detection, should perform rather instantaneous. +- The backward compatibility with Chardet has been greatly improved. The legacy detect function returns an identical charset name whenever possible. +- The detection mechanism has been slightly improved, now Turkish content is detected correctly (most of the time) +- The program has been rewritten to ease the readability and maintainability. (+Using static typing)+ +- utf_7 detection has been reinstated. + +### Removed +- This package no longer require anything when used with Python 3.5 (Dropped cached_property) +- Removed support for these languages: Catalan, Esperanto, Kazakh, Baque, Volapük, Azeri, Galician, Nynorsk, Macedonian, and Serbocroatian. +- The exception hook on UnicodeDecodeError has been removed. + +### Deprecated +- Methods coherence_non_latin, w_counter, chaos_secondary_pass of the class CharsetMatch are now deprecated and scheduled for removal in v3.0 + +### Fixed +- The CLI output used the relative path of the file(s). Should be absolute. + +## [1.4.1](https://github.com/Ousret/charset_normalizer/compare/1.4.0...1.4.1) (2021-05-28) +### Fixed +- Logger configuration/usage no longer conflict with others (PR #44) + +## [1.4.0](https://github.com/Ousret/charset_normalizer/compare/1.3.9...1.4.0) (2021-05-21) +### Removed +- Using standard logging instead of using the package loguru. +- Dropping nose test framework in favor of the maintained pytest. +- Choose to not use dragonmapper package to help with gibberish Chinese/CJK text. +- Require cached_property only for Python 3.5 due to constraint. Dropping for every other interpreter version. +- Stop support for UTF-7 that does not contain a SIG. +- Dropping PrettyTable, replaced with pure JSON output in CLI. + +### Fixed +- BOM marker in a CharsetNormalizerMatch instance could be False in rare cases even if obviously present. Due to the sub-match factoring process. +- Not searching properly for the BOM when trying utf32/16 parent codec. + +### Changed +- Improving the package final size by compressing frequencies.json. +- Huge improvement over the larges payload. + +### Added +- CLI now produces JSON consumable output. +- Return ASCII if given sequences fit. Given reasonable confidence. + +## [1.3.9](https://github.com/Ousret/charset_normalizer/compare/1.3.8...1.3.9) (2021-05-13) + +### Fixed +- In some very rare cases, you may end up getting encode/decode errors due to a bad bytes payload (PR #40) + +## [1.3.8](https://github.com/Ousret/charset_normalizer/compare/1.3.7...1.3.8) (2021-05-12) + +### Fixed +- Empty given payload for detection may cause an exception if trying to access the `alphabets` property. (PR #39) + +## [1.3.7](https://github.com/Ousret/charset_normalizer/compare/1.3.6...1.3.7) (2021-05-12) + +### Fixed +- The legacy detect function should return UTF-8-SIG if sig is present in the payload. (PR #38) + +## [1.3.6](https://github.com/Ousret/charset_normalizer/compare/1.3.5...1.3.6) (2021-02-09) + +### Changed +- Amend the previous release to allow prettytable 2.0 (PR #35) + +## [1.3.5](https://github.com/Ousret/charset_normalizer/compare/1.3.4...1.3.5) (2021-02-08) + +### Fixed +- Fix error while using the package with a python pre-release interpreter (PR #33) + +### Changed +- Dependencies refactoring, constraints revised. + +### Added +- Add python 3.9 and 3.10 to the supported interpreters diff --git a/tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md b/tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..0754df89c --- /dev/null +++ b/tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at ahmed.tahri@cloudnursery.dev. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/tools/retro-library/charset_normalizer/CONTRIBUTING.md b/tools/retro-library/charset_normalizer/CONTRIBUTING.md new file mode 100644 index 000000000..abee674bd --- /dev/null +++ b/tools/retro-library/charset_normalizer/CONTRIBUTING.md @@ -0,0 +1,72 @@ +# Contribution Guidelines + +If you’re reading this, you’re probably interested in contributing to Charset Normalizer. +Thank you very much! Open source projects live-and-die based on the support they receive from others, +and the fact that you’re even considering contributing to this project is very generous of you. + +## Questions + +The GitHub issue tracker is for *bug reports* and *feature requests*. +Questions are allowed only when no answer are provided in docs. + +## Good Bug Reports + +Please be aware of the following things when filing bug reports: + +1. Avoid raising duplicate issues. *Please* use the GitHub issue search feature + to check whether your bug report or feature request has been mentioned in + the past. Duplicate bug reports and feature requests are a huge maintenance + burden on the limited resources of the project. If it is clear from your + report that you would have struggled to find the original, that's ok, but + if searching for a selection of words in your issue title would have found + the duplicate then the issue will likely be closed extremely abruptly. +2. When filing bug reports about exceptions or tracebacks, please include the + *complete* traceback. Partial tracebacks, or just the exception text, are + not helpful. Issues that do not contain complete tracebacks may be closed + without warning. +3. Make sure you provide a suitable amount of information to work with. This + means you should provide: + + - Guidance on **how to reproduce the issue**. Ideally, this should be a + *small* code sample that can be run immediately by the maintainers. + Failing that, let us know what you're doing, how often it happens, what + environment you're using, etc. Be thorough: it prevents us needing to ask + further questions. + - Tell us **what you expected to happen**. When we run your example code, + what are we expecting to happen? What does "success" look like for your + code? + - Tell us **what actually happens**. It's not helpful for you to say "it + doesn't work" or "it fails". Tell us *how* it fails: do you get an + exception? A None answer? How was the actual result + different from your expected result? + - Tell us **what version of Charset Normalizer you're using**, and + **how you installed it**. Different versions of Charset Normalizer behave + differently and have different bugs. + + If you do not provide all of these things, it will take us much longer to + fix your problem. If we ask you to clarify these, and you never respond, we + will close your issue without fixing it. + + +## What PR are we accepting? + +Mostly anything, from cosmetic to the detection-mechanism improvement at the solo condition that you do not break +the backward-compatibility. + +## What PR may be doomed? + + - Add support for a Python unsupported charset/encoding +> If you looked carefully at the project, you would see that it aims to be generic whenever possible. So adding a specific prober is out of the question. + + - Of course, if the CI/CD are failing +> Getting the discussion started often mean doing the minimum effort to get it Green! (Be reassured, maintainers will look into it, given a reasonable amount of time) + + - Submitting a PR without any description OR viable commit description +> This is obvious, maintainers need to understand as fast as possible what are you trying to submit without putting too much effort. + +## How to run tests locally? + +It is essential that you run, prior to any submissions the mandatory checks. +Run the script `./bin/run_checks.sh` to verify that your modification are not breaking anything. + +Also, make sure to run the `./bin/run_autofix.sh` to comply with the style format and import sorting. diff --git a/tools/retro-library/charset_normalizer/LICENSE b/tools/retro-library/charset_normalizer/LICENSE new file mode 100644 index 000000000..ad82355b8 --- /dev/null +++ b/tools/retro-library/charset_normalizer/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 TAHRI Ahmed R. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/MANIFEST.in b/tools/retro-library/charset_normalizer/MANIFEST.in new file mode 100644 index 000000000..3792f5bb9 --- /dev/null +++ b/tools/retro-library/charset_normalizer/MANIFEST.in @@ -0,0 +1,5 @@ +include LICENSE README.md CHANGELOG.md charset_normalizer/py.typed dev-requirements.txt +recursive-include data *.md +recursive-include data *.txt +recursive-include docs * +recursive-include tests * diff --git a/tools/retro-library/charset_normalizer/README.md b/tools/retro-library/charset_normalizer/README.md new file mode 100644 index 000000000..13e6e14ff --- /dev/null +++ b/tools/retro-library/charset_normalizer/README.md @@ -0,0 +1,257 @@ +

Charset Detection, for Everyone 👋

+ +

+ The Real First Universal Charset Detector
+ + + + + Download Count Total + + + + +

+

+ Featured Packages
+ + Static Badge + + + Static Badge + +

+

+ In other language (unofficial port - by the community)
+ + Static Badge + +

+ +> A library that helps you read text from an unknown charset encoding.
Motivated by `chardet`, +> I'm trying to resolve the issue by taking a new approach. +> All IANA character set names for which the Python core library provides codecs are supported. + +

+ >>>>> 👉 Try Me Online Now, Then Adopt Me 👈 <<<<< +

+ +This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**. + +| Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) | +|--------------------------------------------------|:---------------------------------------------:|:--------------------------------------------------------------------------------------------------:|:-----------------------------------------------:| +| `Fast` | ❌ | ✅ | ✅ | +| `Universal**` | ❌ | ✅ | ❌ | +| `Reliable` **without** distinguishable standards | ❌ | ✅ | ✅ | +| `Reliable` **with** distinguishable standards | ✅ | ✅ | ✅ | +| `License` | LGPL-2.1
_restrictive_ | MIT | MPL-1.1
_restrictive_ | +| `Native Python` | ✅ | ✅ | ❌ | +| `Detect spoken language` | ❌ | ✅ | N/A | +| `UnicodeDecodeError Safety` | ❌ | ✅ | ❌ | +| `Whl Size (min)` | 193.6 kB | 42 kB | ~200 kB | +| `Supported Encoding` | 33 | 🎉 [99](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 | + +

+Reading Normalized TextCat Reading Text +

+ +*\*\* : They are clearly using specific code for a specific encoding even if covering most of used one*
+Did you got there because of the logs? See [https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html](https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html) + +## ⚡ Performance + +This package offer better performance than its counterpart Chardet. Here are some numbers. + +| Package | Accuracy | Mean per file (ms) | File per sec (est) | +|-----------------------------------------------|:--------:|:------------------:|:------------------:| +| [chardet](https://github.com/chardet/chardet) | 86 % | 200 ms | 5 file/sec | +| charset-normalizer | **98 %** | **10 ms** | 100 file/sec | + +| Package | 99th percentile | 95th percentile | 50th percentile | +|-----------------------------------------------|:---------------:|:---------------:|:---------------:| +| [chardet](https://github.com/chardet/chardet) | 1200 ms | 287 ms | 23 ms | +| charset-normalizer | 100 ms | 50 ms | 5 ms | + +Chardet's performance on larger file (1MB+) are very poor. Expect huge difference on large payload. + +> Stats are generated using 400+ files using default parameters. More details on used files, see GHA workflows. +> And yes, these results might change at any time. The dataset can be updated to include more files. +> The actual delays heavily depends on your CPU capabilities. The factors should remain the same. +> Keep in mind that the stats are generous and that Chardet accuracy vs our is measured using Chardet initial capability +> (eg. Supported Encoding) Challenge-them if you want. + +## ✨ Installation + +Using pip: + +```sh +pip install charset-normalizer -U +``` + +## 🚀 Basic Usage + +### CLI +This package comes with a CLI. + +``` +usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD] + file [file ...] + +The Real First Universal Charset Detector. Discover originating encoding used +on text file. Normalize text to unicode. + +positional arguments: + files File(s) to be analysed + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display complementary information about file if any. + Stdout will contain logs about the detection process. + -a, --with-alternative + Output complementary possibilities if any. Top-level + JSON WILL be a list. + -n, --normalize Permit to normalize input file. If not set, program + does not write anything. + -m, --minimal Only output the charset detected to STDOUT. Disabling + JSON output. + -r, --replace Replace file when trying to normalize it instead of + creating a new one. + -f, --force Replace file without asking if you are sure, use this + flag with caution. + -t THRESHOLD, --threshold THRESHOLD + Define a custom maximum amount of chaos allowed in + decoded content. 0. <= chaos <= 1. + --version Show version information and exit. +``` + +```bash +normalizer ./data/sample.1.fr.srt +``` + +or + +```bash +python -m charset_normalizer ./data/sample.1.fr.srt +``` + +🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format. + +```json +{ + "path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt", + "encoding": "cp1252", + "encoding_aliases": [ + "1252", + "windows_1252" + ], + "alternative_encodings": [ + "cp1254", + "cp1256", + "cp1258", + "iso8859_14", + "iso8859_15", + "iso8859_16", + "iso8859_3", + "iso8859_9", + "latin_1", + "mbcs" + ], + "language": "French", + "alphabets": [ + "Basic Latin", + "Latin-1 Supplement" + ], + "has_sig_or_bom": false, + "chaos": 0.149, + "coherence": 97.152, + "unicode_path": null, + "is_preferred": true +} +``` + +### Python +*Just print out normalized text* +```python +from charset_normalizer import from_path + +results = from_path('./my_subtitle.srt') + +print(str(results.best())) +``` + +*Upgrade your code without effort* +```python +from charset_normalizer import detect +``` + +The above code will behave the same as **chardet**. We ensure that we offer the best (reasonable) BC result possible. + +See the docs for advanced usage : [readthedocs.io](https://charset-normalizer.readthedocs.io/en/latest/) + +## 😇 Why + +When I started using Chardet, I noticed that it was not suited to my expectations, and I wanted to propose a +reliable alternative using a completely different method. Also! I never back down on a good challenge! + +I **don't care** about the **originating charset** encoding, because **two different tables** can +produce **two identical rendered string.** +What I want is to get readable text, the best I can. + +In a way, **I'm brute forcing text decoding.** How cool is that ? 😎 + +Don't confuse package **ftfy** with charset-normalizer or chardet. ftfy goal is to repair unicode string whereas charset-normalizer to convert raw file in unknown encoding to unicode. + +## 🍰 How + + - Discard all charset encoding table that could not fit the binary content. + - Measure noise, or the mess once opened (by chunks) with a corresponding charset encoding. + - Extract matches with the lowest mess detected. + - Additionally, we measure coherence / probe for a language. + +**Wait a minute**, what is noise/mess and coherence according to **YOU ?** + +*Noise :* I opened hundred of text files, **written by humans**, with the wrong encoding table. **I observed**, then +**I established** some ground rules about **what is obvious** when **it seems like** a mess. + I know that my interpretation of what is noise is probably incomplete, feel free to contribute in order to + improve or rewrite it. + +*Coherence :* For each language there is on earth, we have computed ranked letter appearance occurrences (the best we can). So I thought +that intel is worth something here. So I use those records against decoded text to check if I can detect intelligent design. + +## ⚡ Known limitations + + - Language detection is unreliable when text contains two or more languages sharing identical letters. (eg. HTML (english tags) + Turkish content (Sharing Latin characters)) + - Every charset detector heavily depends on sufficient content. In common cases, do not bother run detection on very tiny content. + +## ⚠️ About Python EOLs + +**If you are running:** + +- Python >=2.7,<3.5: Unsupported +- Python 3.5: charset-normalizer < 2.1 +- Python 3.6: charset-normalizer < 3.1 +- Python 3.7: charset-normalizer < 4.0 + +Upgrade your Python interpreter as soon as possible. + +## 👤 Contributing + +Contributions, issues and feature requests are very much welcome.
+Feel free to check [issues page](https://github.com/ousret/charset_normalizer/issues) if you want to contribute. + +## 📝 License + +Copyright © [Ahmed TAHRI @Ousret](https://github.com/Ousret).
+This project is [MIT](https://github.com/Ousret/charset_normalizer/blob/master/LICENSE) licensed. + +Characters frequencies used in this project © 2012 [Denny Vrandečić](http://simia.net/letters/) + +## 💼 For Enterprise + +Professional support for charset-normalizer is available as part of the [Tidelift +Subscription][1]. Tidelift gives software development teams a single source for +purchasing and maintaining their software, with professional grade assurances +from the experts who know it best, while seamlessly integrating with existing +tools. + +[1]: https://tidelift.com/subscription/pkg/pypi-charset-normalizer?utm_source=pypi-charset-normalizer&utm_medium=readme diff --git a/tools/retro-library/charset_normalizer/SECURITY.md b/tools/retro-library/charset_normalizer/SECURITY.md new file mode 100644 index 000000000..005467cec --- /dev/null +++ b/tools/retro-library/charset_normalizer/SECURITY.md @@ -0,0 +1,4 @@ +# Security Disclosures + +To report a security vulnerability, please use the [Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure with maintainers. diff --git a/tools/retro-library/charset_normalizer/UPGRADE.md b/tools/retro-library/charset_normalizer/UPGRADE.md new file mode 100644 index 000000000..4b8f7bb14 --- /dev/null +++ b/tools/retro-library/charset_normalizer/UPGRADE.md @@ -0,0 +1,31 @@ +Guide to upgrade your code from v1 to v2 +---------------------------------------- + + * If you are using the legacy `detect` function, that is it. You have nothing to do. + +## Detection + +### Before + +```python +from charset_normalizer import CharsetNormalizerMatches + +results = CharsetNormalizerMatches.from_bytes( + '我没有埋怨，磋砣的只是一些时间。'.encode('utf_32') +) +``` + +### After + +```python +from charset_normalizer import from_bytes + +results = from_bytes( + '我没有埋怨，磋砣的只是一些时间。'.encode('utf_32') +) +``` + +Methods that once were staticmethods of the class `CharsetNormalizerMatches` are now basic functions. +`from_fp`, `from_bytes`, `from_fp` and `` are concerned. + +Staticmethods scheduled to be removed in version 3.0 diff --git a/tools/retro-library/charset_normalizer/bin/bc.py b/tools/retro-library/charset_normalizer/bin/bc.py new file mode 100644 index 000000000..4eacc1c49 --- /dev/null +++ b/tools/retro-library/charset_normalizer/bin/bc.py @@ -0,0 +1,109 @@ +#!/bin/python +from __future__ import annotations + +import argparse +from glob import glob +from os.path import isdir +from sys import argv + +from chardet import detect as chardet_detect + +from charset_normalizer import detect as tbt_detect +from charset_normalizer.utils import iana_name + + +def calc_equivalence(content: bytes, cp_a: str, cp_b: str): + try: + str_a = content.decode(cp_a) + str_b = content.decode(cp_b) + except UnicodeDecodeError: + return 0.0 + + character_count = len(str_a) + diff_character_count = sum(chr_a != chr_b for chr_a, chr_b in zip(str_a, str_b)) + + return 1.0 - (diff_character_count / character_count) + + +def cli_bc(arguments: list[str]): + parser = argparse.ArgumentParser( + description="BC script checker for Charset-Normalizer with Chardet" + ) + + parser.add_argument( + "-c", + "--coverage", + action="store", + default=85, + type=int, + dest="coverage", + help="Define the minimum acceptable coverage to succeed", + ) + + args = parser.parse_args(arguments) + + if not isdir("./char-dataset"): + print( + "This script require https://github.com/Ousret/char-dataset to be cloned on package root directory" + ) + exit(1) + + success_count = 0 + total_count = 0 + + for tbt_path in sorted(glob("./char-dataset/**/*.*")): + total_count += 1 + + with open(tbt_path, "rb") as fp: + content = fp.read() + + chardet_result = chardet_detect(content) + chardet_encoding = chardet_result["encoding"] + + charset_normalizer_result = tbt_detect(content) + charset_normalizer_encoding = charset_normalizer_result["encoding"] + + if [chardet_encoding, charset_normalizer_encoding].count(None) == 1: + print( + f"⚡⚡ '{tbt_path}' (BC-Break) New('{charset_normalizer_encoding}') vs Legacy('{chardet_encoding}')" + ) + continue + + if charset_normalizer_encoding == chardet_encoding: + success_count += 1 + print(f"✅✅ '{tbt_path}' (BC)") + continue + + if (chardet_encoding is None and charset_normalizer_encoding is None) or ( + iana_name(chardet_encoding, False) + == iana_name(charset_normalizer_encoding, False) + ): + success_count += 1 + print(f"✅✅ '{tbt_path}' (BC)") + continue + + calc_eq = calc_equivalence( + content, chardet_encoding, charset_normalizer_encoding + ) + + if calc_eq >= 0.98: + success_count += 1 + print( + f"️✅ ️'{tbt_path}' (got '{charset_normalizer_encoding}' but " + f"eq {chardet_encoding} WITH {round(calc_eq * 100., 3)} %)" + ) + continue + + print( + f"⚡⚡ '{tbt_path}' (BC-Break) New('{charset_normalizer_encoding}') vs Legacy('{chardet_encoding}')" + ) + + success_ratio = round(success_count / total_count, 2) * 100.0 + + print(f"Total EST BC = {success_ratio} % ({success_count} / {total_count} files)") + + return 0 if success_ratio >= args.coverage else 1 + + +if __name__ == "__main__": + exit(cli_bc(argv[1:])) diff --git a/tools/retro-library/charset_normalizer/bin/coverage.py b/tools/retro-library/charset_normalizer/bin/coverage.py new file mode 100644 index 000000000..e5ba0110e --- /dev/null +++ b/tools/retro-library/charset_normalizer/bin/coverage.py @@ -0,0 +1,106 @@ +#!/bin/python +from __future__ import annotations + +import argparse +from glob import glob +from os import sep +from os.path import isdir +from sys import argv + +from charset_normalizer import __version__, from_path +from charset_normalizer.utils import iana_name + + +def calc_equivalence(content: bytes, cp_a: str, cp_b: str): + str_a = content.decode(cp_a) + str_b = content.decode(cp_b) + + character_count = len(str_a) + diff_character_count = sum(chr_a != chr_b for chr_a, chr_b in zip(str_a, str_b)) + + return 1.0 - (diff_character_count / character_count) + + +def cli_coverage(arguments: list[str]): + parser = argparse.ArgumentParser( + description="Embedded detection success coverage script checker for Charset-Normalizer" + ) + + parser.add_argument( + "-p", + "--with-preemptive", + action="store_true", + default=False, + dest="preemptive", + help="Enable the preemptive scan behaviour during coverage check", + ) + parser.add_argument( + "-c", + "--coverage", + action="store", + default=90, + type=int, + dest="coverage", + help="Define the minimum acceptable coverage to succeed", + ) + + args = parser.parse_args(arguments) + + if not isdir("./char-dataset"): + print( + "This script require https://github.com/Ousret/char-dataset to be cloned on package root directory" + ) + exit(1) + + print(f"> using charset-normalizer {__version__}") + + success_count = 0 + total_count = 0 + + for tbt_path in sorted(glob("./char-dataset/**/*.*")): + expected_encoding = tbt_path.split(sep)[-2] + total_count += 1 + + results = from_path(tbt_path, preemptive_behaviour=args.preemptive) + + if expected_encoding == "None" and len(results) == 0: + print(f"✅✅ '{tbt_path}'") + success_count += 1 + continue + + if len(results) == 0: + print(f"⚡⚡ '{tbt_path}' (nothing)") + continue + + result = results.best() + + if ( + expected_encoding in result.could_be_from_charset + or iana_name(expected_encoding) in result.could_be_from_charset + ): + print(f"✅✅ '{tbt_path}'") + success_count += 1 + continue + + calc_eq = calc_equivalence(result.raw, expected_encoding, result.encoding) + + if calc_eq >= 0.98: + success_count += 1 + print( + f"️✅ ️'{tbt_path}' (got '{result.encoding}' but equivalence {round(calc_eq * 100., 3)} %)" + ) + continue + + print(f"⚡ '{tbt_path}' (got '{result.encoding}')") + + success_ratio = round(success_count / total_count, 2) * 100.0 + + print( + f"Total EST coverage = {success_ratio} % ({success_count} / {total_count} files)" + ) + + return 0 if success_ratio >= args.coverage else 1 + + +if __name__ == "__main__": + exit(cli_coverage(argv[1:])) diff --git a/tools/retro-library/charset_normalizer/bin/integration.py b/tools/retro-library/charset_normalizer/bin/integration.py new file mode 100644 index 000000000..7313ae5e0 --- /dev/null +++ b/tools/retro-library/charset_normalizer/bin/integration.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from requests import __version__, get + +from charset_normalizer import __version__ as __version_cn__ +from charset_normalizer import detect + +if __name__ == "__main__": + print(f"requests {__version__}") + print(f"charset_normalizer {__version_cn__}") + + files: list[str] = get("http://127.0.0.1:8080/").json() + + print("## Testing with actual files") + + for file in files: + r = get("http://127.0.0.1:8080/" + file) + + if r.ok is False: + print(f"Unable to retrieve '{file}' | HTTP/{r.status_code}") + exit(1) + + expected_encoding = detect(r.content)["encoding"] + + if expected_encoding != r.apparent_encoding: + print( + f"Integration test failed | File '{file}' | Expected '{expected_encoding}' got '{r.apparent_encoding}'" + ) + exit(1) + + print(f"✅✅ '{file}' OK") + + print("## Testing with edge cases") + + # Should NOT crash + get("http://127.0.0.1:8080/edge/empty/json").json() + + print("✅✅ Empty JSON OK") + + if get("http://127.0.0.1:8080/edge/empty/plain").apparent_encoding != "utf-8": + print("Empty payload SHOULD not return apparent_encoding != UTF-8") + exit(1) + + print("✅✅ Empty Plain Text OK") + + r = get("http://127.0.0.1:8080/edge/gb18030/json") + + if r.apparent_encoding != "GB18030": + print("JSON Basic Detection FAILURE (/edge/gb18030/json)") + exit(1) + + r.json() + + print("✅✅ GB18030 JSON Encoded OK") + + print("Integration tests passed!") diff --git a/tools/retro-library/charset_normalizer/bin/performance.py b/tools/retro-library/charset_normalizer/bin/performance.py new file mode 100644 index 000000000..41195b8f8 --- /dev/null +++ b/tools/retro-library/charset_normalizer/bin/performance.py @@ -0,0 +1,155 @@ +#!/bin/python +from __future__ import annotations + +import argparse +from glob import glob +from math import ceil +from os.path import isdir +from statistics import mean, stdev +from sys import argv +from time import perf_counter_ns + +from chardet import detect as chardet_detect + +from charset_normalizer import detect + + +def calc_percentile(data, percentile): + n = len(data) + p = n * percentile / 100 + sorted_data = sorted(data) + + return sorted_data[int(p)] if p.is_integer() else sorted_data[int(ceil(p)) - 1] + + +def performance_compare(arguments): + parser = argparse.ArgumentParser( + description="Performance CI/CD check for Charset-Normalizer" + ) + + parser.add_argument( + "-s", + "--size-increase", + action="store", + default=1, + type=int, + dest="size_coeff", + help="Apply artificial size increase to challenge the detection mechanism further", + ) + + args = parser.parse_args(arguments) + + if not isdir("./char-dataset"): + print( + "This script require https://github.com/Ousret/char-dataset to be cloned on package root directory" + ) + exit(1) + + chardet_results = [] + charset_normalizer_results = [] + + file_list = sorted(glob("./char-dataset/**/*.*")) + total_files = len(file_list) + + for idx, tbt_path in enumerate(file_list): + with open(tbt_path, "rb") as fp: + content = fp.read() * args.size_coeff + + before = perf_counter_ns() + chardet_detect(content) + chardet_time = round((perf_counter_ns() - before) / 1000000000, 5) + chardet_results.append(chardet_time) + + before = perf_counter_ns() + detect(content) + charset_normalizer_time = round((perf_counter_ns() - before) / 1000000000, 5) + charset_normalizer_results.append(charset_normalizer_time) + + charset_normalizer_time = charset_normalizer_time or 0.000005 + cn_faster = (chardet_time / charset_normalizer_time) * 100 - 100 + print( + f"{idx + 1:>3}/{total_files} {tbt_path:<82} C:{chardet_time:.5f} " + f"CN:{charset_normalizer_time:.5f} {cn_faster:.1f} %" + ) + + # Print the top 10 rows with the slowest execution time + print( + f"\n{'-' * 102}\nTop 10 rows with the slowest execution time of charset_normalizer:\n" + ) + sorted_results = sorted( + enumerate(charset_normalizer_results), key=lambda x: x[1], reverse=True + ) + for idx, time in sorted_results[:10]: + tbt_path = file_list[idx] + print(f"{idx + 1:>3}/{total_files} {tbt_path:<82} CN:{time:.5f}") + + # Print charset normalizer statistics + min_time = min(charset_normalizer_results) + max_time = max(charset_normalizer_results) + stdev_time = stdev(charset_normalizer_results) + mean_time = mean(charset_normalizer_results) + cv = (stdev_time / mean_time) * 100 # Coefficient of variation + print(f"\n{'-' * 102}\nCharset Normalizer statistics:\n") + print(f"Minimum Execution Time: {min_time:.5f} seconds") + print(f"Maximum Execution Time: {max_time:.5f} seconds") + print(f"Mean Execution Time: {mean_time:.5f} seconds") + print(f"Standard Deviation: {stdev_time:.5f} seconds") + print(f"Coefficient of Variation (CV): {cv:.1f} %") + + # Print comparison statistics for chardet and charset normalizer + chardet_avg_delay = round(mean(chardet_results) * 1000) + chardet_99p = round(calc_percentile(chardet_results, 99) * 1000) + chardet_95p = round(calc_percentile(chardet_results, 95) * 1000) + chardet_50p = round(calc_percentile(chardet_results, 50) * 1000) + + charset_normalizer_avg_delay = round(mean(charset_normalizer_results) * 1000) + charset_normalizer_99p = round( + calc_percentile(charset_normalizer_results, 99) * 1000 + ) + charset_normalizer_95p = round( + calc_percentile(charset_normalizer_results, 95) * 1000 + ) + charset_normalizer_50p = round( + calc_percentile(charset_normalizer_results, 50) * 1000 + ) + + # mypyc can offer performance ~1ms in the 50p. When eq to 0 assume 1 due to imprecise nature of this test. + if charset_normalizer_50p == 0: + charset_normalizer_50p = 1 + + print(f"\n{'-' * 102}\nCharset Normalizer vs Chardet statistics:\n") + + print("------------------------------") + print("--> Chardet Conclusions") + print(" --> Avg: " + str(chardet_avg_delay) + "ms") + print(" --> 99th: " + str(chardet_99p) + "ms") + print(" --> 95th: " + str(chardet_95p) + "ms") + print(" --> 50th: " + str(chardet_50p) + "ms") + + print("------------------------------") + print("--> Charset-Normalizer Conclusions") + print(" --> Avg: " + str(charset_normalizer_avg_delay) + "ms") + print(" --> 99th: " + str(charset_normalizer_99p) + "ms") + print(" --> 95th: " + str(charset_normalizer_95p) + "ms") + print(" --> 50th: " + str(charset_normalizer_50p) + "ms") + + print("------------------------------") + print("--> Charset-Normalizer / Chardet: Performance Сomparison") + print( + " --> Avg: x" + + str(round(chardet_avg_delay / charset_normalizer_avg_delay, 2)) + ) + print(" --> 99th: x" + str(round(chardet_99p / charset_normalizer_99p, 2))) + print(" --> 95th: x" + str(round(chardet_95p / charset_normalizer_95p, 2))) + print(" --> 50th: x" + str(round(chardet_50p / charset_normalizer_50p, 2))) + + return ( + 0 + if chardet_avg_delay > charset_normalizer_avg_delay + and chardet_99p > charset_normalizer_99p + else 1 + ) + + +if __name__ == "__main__": + exit(performance_compare(argv[1:])) diff --git a/tools/retro-library/charset_normalizer/bin/serve.py b/tools/retro-library/charset_normalizer/bin/serve.py new file mode 100644 index 000000000..99d6b6ea4 --- /dev/null +++ b/tools/retro-library/charset_normalizer/bin/serve.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from glob import glob + +from flask import Flask, jsonify, send_from_directory + +app = Flask(__name__) + + +@app.route("/raw/") +def read_file(path): + return ( + send_from_directory("../char-dataset", path, as_attachment=True), + 200, + {"Content-Type": "text/plain"}, + ) + + +@app.route("/") +def read_targets(): + return jsonify( + [ + el.replace("./char-dataset", "/raw").replace("\\", "/") + for el in sorted(glob("./char-dataset/**/*")) + ] + ) + + +@app.route("/edge/empty/plain") +def read_empty_response_plain(): + return b"", 200, {"Content-Type": "text/plain"} + + +@app.route("/edge/empty/json") +def read_empty_response_json(): + return b"{}", 200, {"Content-Type": "application/json"} + + +@app.route("/edge/gb18030/json") +def read_gb18030_response_json(): + return ( + '{"abc": "我没有埋怨，磋砣的只是一些时间。。今觀俗士之論也，以族舉德，以位命賢，茲可謂得論之一體矣，而未獲至論之淑真也。"}'.encode( + "gb18030" + ), + 200, + {"Content-Type": "application/json"}, + ) + + +if __name__ == "__main__": + app.run(host="127.0.0.1", port=8080) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/__init__.py b/tools/retro-library/charset_normalizer/charset_normalizer/__init__.py new file mode 100644 index 000000000..348341fb7 --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/__init__.py @@ -0,0 +1,47 @@ +""" +Charset-Normalizer +~~~~~~~~~~~~~~ +The Real First Universal Charset Detector. +A library that helps you read text from an unknown charset encoding. +Motivated by chardet, This package is trying to resolve the issue by taking a new approach. +All IANA character set names for which the Python core library provides codecs are supported. + +Basic usage: + >>> from charset_normalizer import from_bytes + >>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8')) + >>> best_guess = results.best() + >>> str(best_guess) + 'Bсеки човек има право на образование. Oбразованието!' + +Others methods and usages are available - see the full documentation +at . +:copyright: (c) 2021 by Ahmed TAHRI +:license: MIT, see LICENSE for more details. +""" +from __future__ import annotations + +import logging + +from .api import from_bytes, from_fp, from_path, is_binary +from .legacy import detect +from .models import CharsetMatch, CharsetMatches +from .utils import set_logging_handler +from .version import VERSION, __version__ + +__all__ = ( + "from_fp", + "from_path", + "from_bytes", + "is_binary", + "detect", + "CharsetMatch", + "CharsetMatches", + "__version__", + "VERSION", + "set_logging_handler", +) + +# Attach a NullHandler to the top level logger by default +# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library + +logging.getLogger("charset_normalizer").addHandler(logging.NullHandler()) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/__main__.py b/tools/retro-library/charset_normalizer/charset_normalizer/__main__.py new file mode 100644 index 000000000..e0e76f7bf --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/__main__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +from .cli import cli_detect + +if __name__ == "__main__": + cli_detect() diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/api.py b/tools/retro-library/charset_normalizer/charset_normalizer/api.py new file mode 100644 index 000000000..9ffc049d6 --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/api.py @@ -0,0 +1,670 @@ +from __future__ import annotations + +import logging +from os import PathLike +from typing import BinaryIO + +from .cd import ( + coherence_ratio, + encoding_languages, + mb_encoding_languages, + merge_coherence_ratios, +) +from .constant import IANA_SUPPORTED, TOO_BIG_SEQUENCE, TOO_SMALL_SEQUENCE, TRACE +from .md import mess_ratio +from .models import CharsetMatch, CharsetMatches +from .utils import ( + any_specified_encoding, + cut_sequence_chunks, + iana_name, + identify_sig_or_bom, + is_cp_similar, + is_multi_byte_encoding, + should_strip_sig_or_bom, +) + +# Will most likely be controversial +# logging.addLevelName(TRACE, "TRACE") +logger = logging.getLogger("charset_normalizer") +explain_handler = logging.StreamHandler() +explain_handler.setFormatter( + logging.Formatter("%(asctime)s | %(levelname)s | %(message)s") +) + + +def from_bytes( + sequences: bytes | bytearray, + steps: int = 5, + chunk_size: int = 512, + threshold: float = 0.2, + cp_isolation: list[str] | None = None, + cp_exclusion: list[str] | None = None, + preemptive_behaviour: bool = True, + explain: bool = False, + language_threshold: float = 0.1, + enable_fallback: bool = True, +) -> CharsetMatches: + """ + Given a raw bytes sequence, return the best possibles charset usable to render str objects. + If there is no results, it is a strong indicator that the source is binary/not text. + By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence. + And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will. + + The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page + but never take it for granted. Can improve the performance. + + You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that + purpose. + + This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32. + By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain' + toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging. + Custom logging format and handler can be set manually. + """ + + if not isinstance(sequences, (bytearray, bytes)): + raise TypeError( + "Expected object of type bytes or bytearray, got: {}".format( + type(sequences) + ) + ) + + if explain: + previous_logger_level: int = logger.level + logger.addHandler(explain_handler) + logger.setLevel(TRACE) + + length: int = len(sequences) + + if length == 0: + logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.") + if explain: + logger.removeHandler(explain_handler) + logger.setLevel(previous_logger_level or logging.WARNING) + return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")]) + + if cp_isolation is not None: + logger.log( + TRACE, + "cp_isolation is set. use this flag for debugging purpose. " + "limited list of encoding allowed : %s.", + ", ".join(cp_isolation), + ) + cp_isolation = [iana_name(cp, False) for cp in cp_isolation] + else: + cp_isolation = [] + + if cp_exclusion is not None: + logger.log( + TRACE, + "cp_exclusion is set. use this flag for debugging purpose. " + "limited list of encoding excluded : %s.", + ", ".join(cp_exclusion), + ) + cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion] + else: + cp_exclusion = [] + + if length <= (chunk_size * steps): + logger.log( + TRACE, + "override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.", + steps, + chunk_size, + length, + ) + steps = 1 + chunk_size = length + + if steps > 1 and length / steps < chunk_size: + chunk_size = int(length / steps) + + is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE + is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE + + if is_too_small_sequence: + logger.log( + TRACE, + "Trying to detect encoding from a tiny portion of ({}) byte(s).".format( + length + ), + ) + elif is_too_large_sequence: + logger.log( + TRACE, + "Using lazy str decoding because the payload is quite large, ({}) byte(s).".format( + length + ), + ) + + prioritized_encodings: list[str] = [] + + specified_encoding: str | None = ( + any_specified_encoding(sequences) if preemptive_behaviour else None + ) + + if specified_encoding is not None: + prioritized_encodings.append(specified_encoding) + logger.log( + TRACE, + "Detected declarative mark in sequence. Priority +1 given for %s.", + specified_encoding, + ) + + tested: set[str] = set() + tested_but_hard_failure: list[str] = [] + tested_but_soft_failure: list[str] = [] + + fallback_ascii: CharsetMatch | None = None + fallback_u8: CharsetMatch | None = None + fallback_specified: CharsetMatch | None = None + + results: CharsetMatches = CharsetMatches() + + early_stop_results: CharsetMatches = CharsetMatches() + + sig_encoding, sig_payload = identify_sig_or_bom(sequences) + + if sig_encoding is not None: + prioritized_encodings.append(sig_encoding) + logger.log( + TRACE, + "Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.", + len(sig_payload), + sig_encoding, + ) + + prioritized_encodings.append("ascii") + + if "utf_8" not in prioritized_encodings: + prioritized_encodings.append("utf_8") + + for encoding_iana in prioritized_encodings + IANA_SUPPORTED: + if cp_isolation and encoding_iana not in cp_isolation: + continue + + if cp_exclusion and encoding_iana in cp_exclusion: + continue + + if encoding_iana in tested: + continue + + tested.add(encoding_iana) + + decoded_payload: str | None = None + bom_or_sig_available: bool = sig_encoding == encoding_iana + strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom( + encoding_iana + ) + + if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available: + logger.log( + TRACE, + "Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.", + encoding_iana, + ) + continue + if encoding_iana in {"utf_7"} and not bom_or_sig_available: + logger.log( + TRACE, + "Encoding %s won't be tested as-is because detection is unreliable without BOM/SIG.", + encoding_iana, + ) + continue + + try: + is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana) + except (ModuleNotFoundError, ImportError): + logger.log( + TRACE, + "Encoding %s does not provide an IncrementalDecoder", + encoding_iana, + ) + continue + + try: + if is_too_large_sequence and is_multi_byte_decoder is False: + str( + ( + sequences[: int(50e4)] + if strip_sig_or_bom is False + else sequences[len(sig_payload) : int(50e4)] + ), + encoding=encoding_iana, + ) + else: + decoded_payload = str( + ( + sequences + if strip_sig_or_bom is False + else sequences[len(sig_payload) :] + ), + encoding=encoding_iana, + ) + except (UnicodeDecodeError, LookupError) as e: + if not isinstance(e, LookupError): + logger.log( + TRACE, + "Code page %s does not fit given bytes sequence at ALL. %s", + encoding_iana, + str(e), + ) + tested_but_hard_failure.append(encoding_iana) + continue + + similar_soft_failure_test: bool = False + + for encoding_soft_failed in tested_but_soft_failure: + if is_cp_similar(encoding_iana, encoding_soft_failed): + similar_soft_failure_test = True + break + + if similar_soft_failure_test: + logger.log( + TRACE, + "%s is deemed too similar to code page %s and was consider unsuited already. Continuing!", + encoding_iana, + encoding_soft_failed, + ) + continue + + r_ = range( + 0 if not bom_or_sig_available else len(sig_payload), + length, + int(length / steps), + ) + + multi_byte_bonus: bool = ( + is_multi_byte_decoder + and decoded_payload is not None + and len(decoded_payload) < length + ) + + if multi_byte_bonus: + logger.log( + TRACE, + "Code page %s is a multi byte encoding table and it appear that at least one character " + "was encoded using n-bytes.", + encoding_iana, + ) + + max_chunk_gave_up: int = int(len(r_) / 4) + + max_chunk_gave_up = max(max_chunk_gave_up, 2) + early_stop_count: int = 0 + lazy_str_hard_failure = False + + md_chunks: list[str] = [] + md_ratios = [] + + try: + for chunk in cut_sequence_chunks( + sequences, + encoding_iana, + r_, + chunk_size, + bom_or_sig_available, + strip_sig_or_bom, + sig_payload, + is_multi_byte_decoder, + decoded_payload, + ): + md_chunks.append(chunk) + + md_ratios.append( + mess_ratio( + chunk, + threshold, + explain is True and 1 <= len(cp_isolation) <= 2, + ) + ) + + if md_ratios[-1] >= threshold: + early_stop_count += 1 + + if (early_stop_count >= max_chunk_gave_up) or ( + bom_or_sig_available and strip_sig_or_bom is False + ): + break + except ( + UnicodeDecodeError + ) as e: # Lazy str loading may have missed something there + logger.log( + TRACE, + "LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s", + encoding_iana, + str(e), + ) + early_stop_count = max_chunk_gave_up + lazy_str_hard_failure = True + + # We might want to check the sequence again with the whole content + # Only if initial MD tests passes + if ( + not lazy_str_hard_failure + and is_too_large_sequence + and not is_multi_byte_decoder + ): + try: + sequences[int(50e3) :].decode(encoding_iana, errors="strict") + except UnicodeDecodeError as e: + logger.log( + TRACE, + "LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s", + encoding_iana, + str(e), + ) + tested_but_hard_failure.append(encoding_iana) + continue + + mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0 + if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up: + tested_but_soft_failure.append(encoding_iana) + logger.log( + TRACE, + "%s was excluded because of initial chaos probing. Gave up %i time(s). " + "Computed mean chaos is %f %%.", + encoding_iana, + early_stop_count, + round(mean_mess_ratio * 100, ndigits=3), + ) + # Preparing those fallbacks in case we got nothing. + if ( + enable_fallback + and encoding_iana in ["ascii", "utf_8", specified_encoding] + and not lazy_str_hard_failure + ): + fallback_entry = CharsetMatch( + sequences, + encoding_iana, + threshold, + False, + [], + decoded_payload, + preemptive_declaration=specified_encoding, + ) + if encoding_iana == specified_encoding: + fallback_specified = fallback_entry + elif encoding_iana == "ascii": + fallback_ascii = fallback_entry + else: + fallback_u8 = fallback_entry + continue + + logger.log( + TRACE, + "%s passed initial chaos probing. Mean measured chaos is %f %%", + encoding_iana, + round(mean_mess_ratio * 100, ndigits=3), + ) + + if not is_multi_byte_decoder: + target_languages: list[str] = encoding_languages(encoding_iana) + else: + target_languages = mb_encoding_languages(encoding_iana) + + if target_languages: + logger.log( + TRACE, + "{} should target any language(s) of {}".format( + encoding_iana, str(target_languages) + ), + ) + + cd_ratios = [] + + # We shall skip the CD when its about ASCII + # Most of the time its not relevant to run "language-detection" on it. + if encoding_iana != "ascii": + for chunk in md_chunks: + chunk_languages = coherence_ratio( + chunk, + language_threshold, + ",".join(target_languages) if target_languages else None, + ) + + cd_ratios.append(chunk_languages) + + cd_ratios_merged = merge_coherence_ratios(cd_ratios) + + if cd_ratios_merged: + logger.log( + TRACE, + "We detected language {} using {}".format( + cd_ratios_merged, encoding_iana + ), + ) + + current_match = CharsetMatch( + sequences, + encoding_iana, + mean_mess_ratio, + bom_or_sig_available, + cd_ratios_merged, + ( + decoded_payload + if ( + is_too_large_sequence is False + or encoding_iana in [specified_encoding, "ascii", "utf_8"] + ) + else None + ), + preemptive_declaration=specified_encoding, + ) + + results.append(current_match) + + if ( + encoding_iana in [specified_encoding, "ascii", "utf_8"] + and mean_mess_ratio < 0.1 + ): + # If md says nothing to worry about, then... stop immediately! + if mean_mess_ratio == 0.0: + logger.debug( + "Encoding detection: %s is most likely the one.", + current_match.encoding, + ) + if explain: + logger.removeHandler(explain_handler) + logger.setLevel(previous_logger_level) + return CharsetMatches([current_match]) + + early_stop_results.append(current_match) + + if ( + len(early_stop_results) + and (specified_encoding is None or specified_encoding in tested) + and "ascii" in tested + and "utf_8" in tested + ): + probable_result: CharsetMatch = early_stop_results.best() # type: ignore[assignment] + logger.debug( + "Encoding detection: %s is most likely the one.", + probable_result.encoding, + ) + if explain: + logger.removeHandler(explain_handler) + logger.setLevel(previous_logger_level) + + return CharsetMatches([probable_result]) + + if encoding_iana == sig_encoding: + logger.debug( + "Encoding detection: %s is most likely the one as we detected a BOM or SIG within " + "the beginning of the sequence.", + encoding_iana, + ) + if explain: + logger.removeHandler(explain_handler) + logger.setLevel(previous_logger_level) + return CharsetMatches([results[encoding_iana]]) + + if len(results) == 0: + if fallback_u8 or fallback_ascii or fallback_specified: + logger.log( + TRACE, + "Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.", + ) + + if fallback_specified: + logger.debug( + "Encoding detection: %s will be used as a fallback match", + fallback_specified.encoding, + ) + results.append(fallback_specified) + elif ( + (fallback_u8 and fallback_ascii is None) + or ( + fallback_u8 + and fallback_ascii + and fallback_u8.fingerprint != fallback_ascii.fingerprint + ) + or (fallback_u8 is not None) + ): + logger.debug("Encoding detection: utf_8 will be used as a fallback match") + results.append(fallback_u8) + elif fallback_ascii: + logger.debug("Encoding detection: ascii will be used as a fallback match") + results.append(fallback_ascii) + + if results: + logger.debug( + "Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.", + results.best().encoding, # type: ignore + len(results) - 1, + ) + else: + logger.debug("Encoding detection: Unable to determine any suitable charset.") + + if explain: + logger.removeHandler(explain_handler) + logger.setLevel(previous_logger_level) + + return results + + +def from_fp( + fp: BinaryIO, + steps: int = 5, + chunk_size: int = 512, + threshold: float = 0.20, + cp_isolation: list[str] | None = None, + cp_exclusion: list[str] | None = None, + preemptive_behaviour: bool = True, + explain: bool = False, + language_threshold: float = 0.1, + enable_fallback: bool = True, +) -> CharsetMatches: + """ + Same thing than the function from_bytes but using a file pointer that is already ready. + Will not close the file pointer. + """ + return from_bytes( + fp.read(), + steps, + chunk_size, + threshold, + cp_isolation, + cp_exclusion, + preemptive_behaviour, + explain, + language_threshold, + enable_fallback, + ) + + +def from_path( + path: str | bytes | PathLike, # type: ignore[type-arg] + steps: int = 5, + chunk_size: int = 512, + threshold: float = 0.20, + cp_isolation: list[str] | None = None, + cp_exclusion: list[str] | None = None, + preemptive_behaviour: bool = True, + explain: bool = False, + language_threshold: float = 0.1, + enable_fallback: bool = True, +) -> CharsetMatches: + """ + Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. + Can raise IOError. + """ + with open(path, "rb") as fp: + return from_fp( + fp, + steps, + chunk_size, + threshold, + cp_isolation, + cp_exclusion, + preemptive_behaviour, + explain, + language_threshold, + enable_fallback, + ) + + +def is_binary( + fp_or_path_or_payload: PathLike | str | BinaryIO | bytes, # type: ignore[type-arg] + steps: int = 5, + chunk_size: int = 512, + threshold: float = 0.20, + cp_isolation: list[str] | None = None, + cp_exclusion: list[str] | None = None, + preemptive_behaviour: bool = True, + explain: bool = False, + language_threshold: float = 0.1, + enable_fallback: bool = False, +) -> bool: + """ + Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string. + Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match + are disabled to be stricter around ASCII-compatible but unlikely to be a string. + """ + if isinstance(fp_or_path_or_payload, (str, PathLike)): + guesses = from_path( + fp_or_path_or_payload, + steps=steps, + chunk_size=chunk_size, + threshold=threshold, + cp_isolation=cp_isolation, + cp_exclusion=cp_exclusion, + preemptive_behaviour=preemptive_behaviour, + explain=explain, + language_threshold=language_threshold, + enable_fallback=enable_fallback, + ) + elif isinstance( + fp_or_path_or_payload, + ( + bytes, + bytearray, + ), + ): + guesses = from_bytes( + fp_or_path_or_payload, + steps=steps, + chunk_size=chunk_size, + threshold=threshold, + cp_isolation=cp_isolation, + cp_exclusion=cp_exclusion, + preemptive_behaviour=preemptive_behaviour, + explain=explain, + language_threshold=language_threshold, + enable_fallback=enable_fallback, + ) + else: + guesses = from_fp( + fp_or_path_or_payload, + steps=steps, + chunk_size=chunk_size, + threshold=threshold, + cp_isolation=cp_isolation, + cp_exclusion=cp_exclusion, + preemptive_behaviour=preemptive_behaviour, + explain=explain, + language_threshold=language_threshold, + enable_fallback=enable_fallback, + ) + + return not guesses diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/cd.py b/tools/retro-library/charset_normalizer/charset_normalizer/cd.py new file mode 100644 index 000000000..71a3ed519 --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/cd.py @@ -0,0 +1,395 @@ +from __future__ import annotations + +import importlib +from codecs import IncrementalDecoder +from collections import Counter +from functools import lru_cache +from typing import Counter as TypeCounter + +from .constant import ( + FREQUENCIES, + KO_NAMES, + LANGUAGE_SUPPORTED_COUNT, + TOO_SMALL_SEQUENCE, + ZH_NAMES, +) +from .md import is_suspiciously_successive_range +from .models import CoherenceMatches +from .utils import ( + is_accentuated, + is_latin, + is_multi_byte_encoding, + is_unicode_range_secondary, + unicode_range, +) + + +def encoding_unicode_range(iana_name: str) -> list[str]: + """ + Return associated unicode ranges in a single byte code page. + """ + if is_multi_byte_encoding(iana_name): + raise OSError("Function not supported on multi-byte code page") + + decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder + + p: IncrementalDecoder = decoder(errors="ignore") + seen_ranges: dict[str, int] = {} + character_count: int = 0 + + for i in range(0x40, 0xFF): + chunk: str = p.decode(bytes([i])) + + if chunk: + character_range: str | None = unicode_range(chunk) + + if character_range is None: + continue + + if is_unicode_range_secondary(character_range) is False: + if character_range not in seen_ranges: + seen_ranges[character_range] = 0 + seen_ranges[character_range] += 1 + character_count += 1 + + return sorted( + [ + character_range + for character_range in seen_ranges + if seen_ranges[character_range] / character_count >= 0.15 + ] + ) + + +def unicode_range_languages(primary_range: str) -> list[str]: + """ + Return inferred languages used with a unicode range. + """ + languages: list[str] = [] + + for language, characters in FREQUENCIES.items(): + for character in characters: + if unicode_range(character) == primary_range: + languages.append(language) + break + + return languages + + +@lru_cache() +def encoding_languages(iana_name: str) -> list[str]: + """ + Single-byte encoding language association. Some code page are heavily linked to particular language(s). + This function does the correspondence. + """ + unicode_ranges: list[str] = encoding_unicode_range(iana_name) + primary_range: str | None = None + + for specified_range in unicode_ranges: + if "Latin" not in specified_range: + primary_range = specified_range + break + + if primary_range is None: + return ["Latin Based"] + + return unicode_range_languages(primary_range) + + +@lru_cache() +def mb_encoding_languages(iana_name: str) -> list[str]: + """ + Multi-byte encoding language association. Some code page are heavily linked to particular language(s). + This function does the correspondence. + """ + if ( + iana_name.startswith("shift_") + or iana_name.startswith("iso2022_jp") + or iana_name.startswith("euc_j") + or iana_name == "cp932" + ): + return ["Japanese"] + if iana_name.startswith("gb") or iana_name in ZH_NAMES: + return ["Chinese"] + if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES: + return ["Korean"] + + return [] + + +@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT) +def get_target_features(language: str) -> tuple[bool, bool]: + """ + Determine main aspects from a supported language if it contains accents and if is pure Latin. + """ + target_have_accents: bool = False + target_pure_latin: bool = True + + for character in FREQUENCIES[language]: + if not target_have_accents and is_accentuated(character): + target_have_accents = True + if target_pure_latin and is_latin(character) is False: + target_pure_latin = False + + return target_have_accents, target_pure_latin + + +def alphabet_languages( + characters: list[str], ignore_non_latin: bool = False +) -> list[str]: + """ + Return associated languages associated to given characters. + """ + languages: list[tuple[str, float]] = [] + + source_have_accents = any(is_accentuated(character) for character in characters) + + for language, language_characters in FREQUENCIES.items(): + target_have_accents, target_pure_latin = get_target_features(language) + + if ignore_non_latin and target_pure_latin is False: + continue + + if target_have_accents is False and source_have_accents: + continue + + character_count: int = len(language_characters) + + character_match_count: int = len( + [c for c in language_characters if c in characters] + ) + + ratio: float = character_match_count / character_count + + if ratio >= 0.2: + languages.append((language, ratio)) + + languages = sorted(languages, key=lambda x: x[1], reverse=True) + + return [compatible_language[0] for compatible_language in languages] + + +def characters_popularity_compare( + language: str, ordered_characters: list[str] +) -> float: + """ + Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language. + The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit). + Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.) + """ + if language not in FREQUENCIES: + raise ValueError(f"{language} not available") + + character_approved_count: int = 0 + FREQUENCIES_language_set = set(FREQUENCIES[language]) + + ordered_characters_count: int = len(ordered_characters) + target_language_characters_count: int = len(FREQUENCIES[language]) + + large_alphabet: bool = target_language_characters_count > 26 + + for character, character_rank in zip( + ordered_characters, range(0, ordered_characters_count) + ): + if character not in FREQUENCIES_language_set: + continue + + character_rank_in_language: int = FREQUENCIES[language].index(character) + expected_projection_ratio: float = ( + target_language_characters_count / ordered_characters_count + ) + character_rank_projection: int = int(character_rank * expected_projection_ratio) + + if ( + large_alphabet is False + and abs(character_rank_projection - character_rank_in_language) > 4 + ): + continue + + if ( + large_alphabet is True + and abs(character_rank_projection - character_rank_in_language) + < target_language_characters_count / 3 + ): + character_approved_count += 1 + continue + + characters_before_source: list[str] = FREQUENCIES[language][ + 0:character_rank_in_language + ] + characters_after_source: list[str] = FREQUENCIES[language][ + character_rank_in_language: + ] + characters_before: list[str] = ordered_characters[0:character_rank] + characters_after: list[str] = ordered_characters[character_rank:] + + before_match_count: int = len( + set(characters_before) & set(characters_before_source) + ) + + after_match_count: int = len( + set(characters_after) & set(characters_after_source) + ) + + if len(characters_before_source) == 0 and before_match_count <= 4: + character_approved_count += 1 + continue + + if len(characters_after_source) == 0 and after_match_count <= 4: + character_approved_count += 1 + continue + + if ( + before_match_count / len(characters_before_source) >= 0.4 + or after_match_count / len(characters_after_source) >= 0.4 + ): + character_approved_count += 1 + continue + + return character_approved_count / len(ordered_characters) + + +def alpha_unicode_split(decoded_sequence: str) -> list[str]: + """ + Given a decoded text sequence, return a list of str. Unicode range / alphabet separation. + Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list; + One containing the latin letters and the other hebrew. + """ + layers: dict[str, str] = {} + + for character in decoded_sequence: + if character.isalpha() is False: + continue + + character_range: str | None = unicode_range(character) + + if character_range is None: + continue + + layer_target_range: str | None = None + + for discovered_range in layers: + if ( + is_suspiciously_successive_range(discovered_range, character_range) + is False + ): + layer_target_range = discovered_range + break + + if layer_target_range is None: + layer_target_range = character_range + + if layer_target_range not in layers: + layers[layer_target_range] = character.lower() + continue + + layers[layer_target_range] += character.lower() + + return list(layers.values()) + + +def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches: + """ + This function merge results previously given by the function coherence_ratio. + The return type is the same as coherence_ratio. + """ + per_language_ratios: dict[str, list[float]] = {} + for result in results: + for sub_result in result: + language, ratio = sub_result + if language not in per_language_ratios: + per_language_ratios[language] = [ratio] + continue + per_language_ratios[language].append(ratio) + + merge = [ + ( + language, + round( + sum(per_language_ratios[language]) / len(per_language_ratios[language]), + 4, + ), + ) + for language in per_language_ratios + ] + + return sorted(merge, key=lambda x: x[1], reverse=True) + + +def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches: + """ + We shall NOT return "English—" in CoherenceMatches because it is an alternative + of "English". This function only keeps the best match and remove the em-dash in it. + """ + index_results: dict[str, list[float]] = dict() + + for result in results: + language, ratio = result + no_em_name: str = language.replace("—", "") + + if no_em_name not in index_results: + index_results[no_em_name] = [] + + index_results[no_em_name].append(ratio) + + if any(len(index_results[e]) > 1 for e in index_results): + filtered_results: CoherenceMatches = [] + + for language in index_results: + filtered_results.append((language, max(index_results[language]))) + + return filtered_results + + return results + + +@lru_cache(maxsize=2048) +def coherence_ratio( + decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None +) -> CoherenceMatches: + """ + Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers. + A layer = Character extraction by alphabets/ranges. + """ + + results: list[tuple[str, float]] = [] + ignore_non_latin: bool = False + + sufficient_match_count: int = 0 + + lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else [] + if "Latin Based" in lg_inclusion_list: + ignore_non_latin = True + lg_inclusion_list.remove("Latin Based") + + for layer in alpha_unicode_split(decoded_sequence): + sequence_frequencies: TypeCounter[str] = Counter(layer) + most_common = sequence_frequencies.most_common() + + character_count: int = sum(o for c, o in most_common) + + if character_count <= TOO_SMALL_SEQUENCE: + continue + + popular_character_ordered: list[str] = [c for c, o in most_common] + + for language in lg_inclusion_list or alphabet_languages( + popular_character_ordered, ignore_non_latin + ): + ratio: float = characters_popularity_compare( + language, popular_character_ordered + ) + + if ratio < threshold: + continue + elif ratio >= 0.8: + sufficient_match_count += 1 + + results.append((language, round(ratio, 4))) + + if sufficient_match_count >= 3: + break + + return sorted( + filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True + ) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/cli/__init__.py b/tools/retro-library/charset_normalizer/charset_normalizer/cli/__init__.py new file mode 100644 index 000000000..543a5a4de --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/cli/__init__.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from .__main__ import cli_detect, query_yes_no + +__all__ = ( + "cli_detect", + "query_yes_no", +) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/cli/__main__.py b/tools/retro-library/charset_normalizer/charset_normalizer/cli/__main__.py new file mode 100644 index 000000000..b4f364be4 --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/cli/__main__.py @@ -0,0 +1,321 @@ +from __future__ import annotations + +import argparse +import sys +from json import dumps +from os.path import abspath, basename, dirname, join, realpath +from platform import python_version +from unicodedata import unidata_version + +import charset_normalizer.md as md_module +from charset_normalizer import from_fp +from charset_normalizer.models import CliDetectionResult +from charset_normalizer.version import __version__ + + +def query_yes_no(question: str, default: str = "yes") -> bool: + """Ask a yes/no question via input() and return their answer. + + "question" is a string that is presented to the user. + "default" is the presumed answer if the user just hits . + It must be "yes" (the default), "no" or None (meaning + an answer is required of the user). + + The "answer" return value is True for "yes" or False for "no". + + Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input + """ + valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + sys.stdout.write(question + prompt) + choice = input().lower() + if default is not None and choice == "": + return valid[default] + elif choice in valid: + return valid[choice] + else: + sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") + + +def cli_detect(argv: list[str] | None = None) -> int: + """ + CLI assistant using ARGV and ArgumentParser + :param argv: + :return: 0 if everything is fine, anything else equal trouble + """ + parser = argparse.ArgumentParser( + description="The Real First Universal Charset Detector. " + "Discover originating encoding used on text file. " + "Normalize text to unicode." + ) + + parser.add_argument( + "files", type=argparse.FileType("rb"), nargs="+", help="File(s) to be analysed" + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + dest="verbose", + help="Display complementary information about file if any. " + "Stdout will contain logs about the detection process.", + ) + parser.add_argument( + "-a", + "--with-alternative", + action="store_true", + default=False, + dest="alternatives", + help="Output complementary possibilities if any. Top-level JSON WILL be a list.", + ) + parser.add_argument( + "-n", + "--normalize", + action="store_true", + default=False, + dest="normalize", + help="Permit to normalize input file. If not set, program does not write anything.", + ) + parser.add_argument( + "-m", + "--minimal", + action="store_true", + default=False, + dest="minimal", + help="Only output the charset detected to STDOUT. Disabling JSON output.", + ) + parser.add_argument( + "-r", + "--replace", + action="store_true", + default=False, + dest="replace", + help="Replace file when trying to normalize it instead of creating a new one.", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + dest="force", + help="Replace file without asking if you are sure, use this flag with caution.", + ) + parser.add_argument( + "-i", + "--no-preemptive", + action="store_true", + default=False, + dest="no_preemptive", + help="Disable looking at a charset declaration to hint the detector.", + ) + parser.add_argument( + "-t", + "--threshold", + action="store", + default=0.2, + type=float, + dest="threshold", + help="Define a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.", + ) + parser.add_argument( + "--version", + action="version", + version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format( + __version__, + python_version(), + unidata_version, + "OFF" if md_module.__file__.lower().endswith(".py") else "ON", + ), + help="Show version information and exit.", + ) + + args = parser.parse_args(argv) + + if args.replace is True and args.normalize is False: + if args.files: + for my_file in args.files: + my_file.close() + print("Use --replace in addition of --normalize only.", file=sys.stderr) + return 1 + + if args.force is True and args.replace is False: + if args.files: + for my_file in args.files: + my_file.close() + print("Use --force in addition of --replace only.", file=sys.stderr) + return 1 + + if args.threshold < 0.0 or args.threshold > 1.0: + if args.files: + for my_file in args.files: + my_file.close() + print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr) + return 1 + + x_ = [] + + for my_file in args.files: + matches = from_fp( + my_file, + threshold=args.threshold, + explain=args.verbose, + preemptive_behaviour=args.no_preemptive is False, + ) + + best_guess = matches.best() + + if best_guess is None: + print( + 'Unable to identify originating encoding for "{}". {}'.format( + my_file.name, + ( + "Maybe try increasing maximum amount of chaos." + if args.threshold < 1.0 + else "" + ), + ), + file=sys.stderr, + ) + x_.append( + CliDetectionResult( + abspath(my_file.name), + None, + [], + [], + "Unknown", + [], + False, + 1.0, + 0.0, + None, + True, + ) + ) + else: + x_.append( + CliDetectionResult( + abspath(my_file.name), + best_guess.encoding, + best_guess.encoding_aliases, + [ + cp + for cp in best_guess.could_be_from_charset + if cp != best_guess.encoding + ], + best_guess.language, + best_guess.alphabets, + best_guess.bom, + best_guess.percent_chaos, + best_guess.percent_coherence, + None, + True, + ) + ) + + if len(matches) > 1 and args.alternatives: + for el in matches: + if el != best_guess: + x_.append( + CliDetectionResult( + abspath(my_file.name), + el.encoding, + el.encoding_aliases, + [ + cp + for cp in el.could_be_from_charset + if cp != el.encoding + ], + el.language, + el.alphabets, + el.bom, + el.percent_chaos, + el.percent_coherence, + None, + False, + ) + ) + + if args.normalize is True: + if best_guess.encoding.startswith("utf") is True: + print( + '"{}" file does not need to be normalized, as it already came from unicode.'.format( + my_file.name + ), + file=sys.stderr, + ) + if my_file.closed is False: + my_file.close() + continue + + dir_path = dirname(realpath(my_file.name)) + file_name = basename(realpath(my_file.name)) + + o_: list[str] = file_name.split(".") + + if args.replace is False: + o_.insert(-1, best_guess.encoding) + if my_file.closed is False: + my_file.close() + elif ( + args.force is False + and query_yes_no( + 'Are you sure to normalize "{}" by replacing it ?'.format( + my_file.name + ), + "no", + ) + is False + ): + if my_file.closed is False: + my_file.close() + continue + + try: + x_[0].unicode_path = join(dir_path, ".".join(o_)) + + with open(x_[0].unicode_path, "wb") as fp: + fp.write(best_guess.output()) + except OSError as e: + print(str(e), file=sys.stderr) + if my_file.closed is False: + my_file.close() + return 2 + + if my_file.closed is False: + my_file.close() + + if args.minimal is False: + print( + dumps( + [el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__, + ensure_ascii=True, + indent=4, + ) + ) + else: + for my_file in args.files: + print( + ", ".join( + [ + el.encoding or "undefined" + for el in x_ + if el.path == abspath(my_file.name) + ] + ) + ) + + return 0 + + +if __name__ == "__main__": + cli_detect() diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/constant.py b/tools/retro-library/charset_normalizer/charset_normalizer/constant.py new file mode 100644 index 000000000..1fb9508d2 --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/constant.py @@ -0,0 +1,1998 @@ +from __future__ import annotations + +from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE +from encodings.aliases import aliases +from re import IGNORECASE +from re import compile as re_compile + +# Contain for each eligible encoding a list of/item bytes SIG/BOM +ENCODING_MARKS: dict[str, bytes | list[bytes]] = { + "utf_8": BOM_UTF8, + "utf_7": [ + b"\x2b\x2f\x76\x38", + b"\x2b\x2f\x76\x39", + b"\x2b\x2f\x76\x2b", + b"\x2b\x2f\x76\x2f", + b"\x2b\x2f\x76\x38\x2d", + ], + "gb18030": b"\x84\x31\x95\x33", + "utf_32": [BOM_UTF32_BE, BOM_UTF32_LE], + "utf_16": [BOM_UTF16_BE, BOM_UTF16_LE], +} + +TOO_SMALL_SEQUENCE: int = 32 +TOO_BIG_SEQUENCE: int = int(10e6) + +UTF8_MAXIMAL_ALLOCATION: int = 1_112_064 + +# Up-to-date Unicode ucd/15.0.0 +UNICODE_RANGES_COMBINED: dict[str, range] = { + "Control character": range(32), + "Basic Latin": range(32, 128), + "Latin-1 Supplement": range(128, 256), + "Latin Extended-A": range(256, 384), + "Latin Extended-B": range(384, 592), + "IPA Extensions": range(592, 688), + "Spacing Modifier Letters": range(688, 768), + "Combining Diacritical Marks": range(768, 880), + "Greek and Coptic": range(880, 1024), + "Cyrillic": range(1024, 1280), + "Cyrillic Supplement": range(1280, 1328), + "Armenian": range(1328, 1424), + "Hebrew": range(1424, 1536), + "Arabic": range(1536, 1792), + "Syriac": range(1792, 1872), + "Arabic Supplement": range(1872, 1920), + "Thaana": range(1920, 1984), + "NKo": range(1984, 2048), + "Samaritan": range(2048, 2112), + "Mandaic": range(2112, 2144), + "Syriac Supplement": range(2144, 2160), + "Arabic Extended-B": range(2160, 2208), + "Arabic Extended-A": range(2208, 2304), + "Devanagari": range(2304, 2432), + "Bengali": range(2432, 2560), + "Gurmukhi": range(2560, 2688), + "Gujarati": range(2688, 2816), + "Oriya": range(2816, 2944), + "Tamil": range(2944, 3072), + "Telugu": range(3072, 3200), + "Kannada": range(3200, 3328), + "Malayalam": range(3328, 3456), + "Sinhala": range(3456, 3584), + "Thai": range(3584, 3712), + "Lao": range(3712, 3840), + "Tibetan": range(3840, 4096), + "Myanmar": range(4096, 4256), + "Georgian": range(4256, 4352), + "Hangul Jamo": range(4352, 4608), + "Ethiopic": range(4608, 4992), + "Ethiopic Supplement": range(4992, 5024), + "Cherokee": range(5024, 5120), + "Unified Canadian Aboriginal Syllabics": range(5120, 5760), + "Ogham": range(5760, 5792), + "Runic": range(5792, 5888), + "Tagalog": range(5888, 5920), + "Hanunoo": range(5920, 5952), + "Buhid": range(5952, 5984), + "Tagbanwa": range(5984, 6016), + "Khmer": range(6016, 6144), + "Mongolian": range(6144, 6320), + "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6400), + "Limbu": range(6400, 6480), + "Tai Le": range(6480, 6528), + "New Tai Lue": range(6528, 6624), + "Khmer Symbols": range(6624, 6656), + "Buginese": range(6656, 6688), + "Tai Tham": range(6688, 6832), + "Combining Diacritical Marks Extended": range(6832, 6912), + "Balinese": range(6912, 7040), + "Sundanese": range(7040, 7104), + "Batak": range(7104, 7168), + "Lepcha": range(7168, 7248), + "Ol Chiki": range(7248, 7296), + "Cyrillic Extended-C": range(7296, 7312), + "Georgian Extended": range(7312, 7360), + "Sundanese Supplement": range(7360, 7376), + "Vedic Extensions": range(7376, 7424), + "Phonetic Extensions": range(7424, 7552), + "Phonetic Extensions Supplement": range(7552, 7616), + "Combining Diacritical Marks Supplement": range(7616, 7680), + "Latin Extended Additional": range(7680, 7936), + "Greek Extended": range(7936, 8192), + "General Punctuation": range(8192, 8304), + "Superscripts and Subscripts": range(8304, 8352), + "Currency Symbols": range(8352, 8400), + "Combining Diacritical Marks for Symbols": range(8400, 8448), + "Letterlike Symbols": range(8448, 8528), + "Number Forms": range(8528, 8592), + "Arrows": range(8592, 8704), + "Mathematical Operators": range(8704, 8960), + "Miscellaneous Technical": range(8960, 9216), + "Control Pictures": range(9216, 9280), + "Optical Character Recognition": range(9280, 9312), + "Enclosed Alphanumerics": range(9312, 9472), + "Box Drawing": range(9472, 9600), + "Block Elements": range(9600, 9632), + "Geometric Shapes": range(9632, 9728), + "Miscellaneous Symbols": range(9728, 9984), + "Dingbats": range(9984, 10176), + "Miscellaneous Mathematical Symbols-A": range(10176, 10224), + "Supplemental Arrows-A": range(10224, 10240), + "Braille Patterns": range(10240, 10496), + "Supplemental Arrows-B": range(10496, 10624), + "Miscellaneous Mathematical Symbols-B": range(10624, 10752), + "Supplemental Mathematical Operators": range(10752, 11008), + "Miscellaneous Symbols and Arrows": range(11008, 11264), + "Glagolitic": range(11264, 11360), + "Latin Extended-C": range(11360, 11392), + "Coptic": range(11392, 11520), + "Georgian Supplement": range(11520, 11568), + "Tifinagh": range(11568, 11648), + "Ethiopic Extended": range(11648, 11744), + "Cyrillic Extended-A": range(11744, 11776), + "Supplemental Punctuation": range(11776, 11904), + "CJK Radicals Supplement": range(11904, 12032), + "Kangxi Radicals": range(12032, 12256), + "Ideographic Description Characters": range(12272, 12288), + "CJK Symbols and Punctuation": range(12288, 12352), + "Hiragana": range(12352, 12448), + "Katakana": range(12448, 12544), + "Bopomofo": range(12544, 12592), + "Hangul Compatibility Jamo": range(12592, 12688), + "Kanbun": range(12688, 12704), + "Bopomofo Extended": range(12704, 12736), + "CJK Strokes": range(12736, 12784), + "Katakana Phonetic Extensions": range(12784, 12800), + "Enclosed CJK Letters and Months": range(12800, 13056), + "CJK Compatibility": range(13056, 13312), + "CJK Unified Ideographs Extension A": range(13312, 19904), + "Yijing Hexagram Symbols": range(19904, 19968), + "CJK Unified Ideographs": range(19968, 40960), + "Yi Syllables": range(40960, 42128), + "Yi Radicals": range(42128, 42192), + "Lisu": range(42192, 42240), + "Vai": range(42240, 42560), + "Cyrillic Extended-B": range(42560, 42656), + "Bamum": range(42656, 42752), + "Modifier Tone Letters": range(42752, 42784), + "Latin Extended-D": range(42784, 43008), + "Syloti Nagri": range(43008, 43056), + "Common Indic Number Forms": range(43056, 43072), + "Phags-pa": range(43072, 43136), + "Saurashtra": range(43136, 43232), + "Devanagari Extended": range(43232, 43264), + "Kayah Li": range(43264, 43312), + "Rejang": range(43312, 43360), + "Hangul Jamo Extended-A": range(43360, 43392), + "Javanese": range(43392, 43488), + "Myanmar Extended-B": range(43488, 43520), + "Cham": range(43520, 43616), + "Myanmar Extended-A": range(43616, 43648), + "Tai Viet": range(43648, 43744), + "Meetei Mayek Extensions": range(43744, 43776), + "Ethiopic Extended-A": range(43776, 43824), + "Latin Extended-E": range(43824, 43888), + "Cherokee Supplement": range(43888, 43968), + "Meetei Mayek": range(43968, 44032), + "Hangul Syllables": range(44032, 55216), + "Hangul Jamo Extended-B": range(55216, 55296), + "High Surrogates": range(55296, 56192), + "High Private Use Surrogates": range(56192, 56320), + "Low Surrogates": range(56320, 57344), + "Private Use Area": range(57344, 63744), + "CJK Compatibility Ideographs": range(63744, 64256), + "Alphabetic Presentation Forms": range(64256, 64336), + "Arabic Presentation Forms-A": range(64336, 65024), + "Variation Selectors": range(65024, 65040), + "Vertical Forms": range(65040, 65056), + "Combining Half Marks": range(65056, 65072), + "CJK Compatibility Forms": range(65072, 65104), + "Small Form Variants": range(65104, 65136), + "Arabic Presentation Forms-B": range(65136, 65280), + "Halfwidth and Fullwidth Forms": range(65280, 65520), + "Specials": range(65520, 65536), + "Linear B Syllabary": range(65536, 65664), + "Linear B Ideograms": range(65664, 65792), + "Aegean Numbers": range(65792, 65856), + "Ancient Greek Numbers": range(65856, 65936), + "Ancient Symbols": range(65936, 66000), + "Phaistos Disc": range(66000, 66048), + "Lycian": range(66176, 66208), + "Carian": range(66208, 66272), + "Coptic Epact Numbers": range(66272, 66304), + "Old Italic": range(66304, 66352), + "Gothic": range(66352, 66384), + "Old Permic": range(66384, 66432), + "Ugaritic": range(66432, 66464), + "Old Persian": range(66464, 66528), + "Deseret": range(66560, 66640), + "Shavian": range(66640, 66688), + "Osmanya": range(66688, 66736), + "Osage": range(66736, 66816), + "Elbasan": range(66816, 66864), + "Caucasian Albanian": range(66864, 66928), + "Vithkuqi": range(66928, 67008), + "Linear A": range(67072, 67456), + "Latin Extended-F": range(67456, 67520), + "Cypriot Syllabary": range(67584, 67648), + "Imperial Aramaic": range(67648, 67680), + "Palmyrene": range(67680, 67712), + "Nabataean": range(67712, 67760), + "Hatran": range(67808, 67840), + "Phoenician": range(67840, 67872), + "Lydian": range(67872, 67904), + "Meroitic Hieroglyphs": range(67968, 68000), + "Meroitic Cursive": range(68000, 68096), + "Kharoshthi": range(68096, 68192), + "Old South Arabian": range(68192, 68224), + "Old North Arabian": range(68224, 68256), + "Manichaean": range(68288, 68352), + "Avestan": range(68352, 68416), + "Inscriptional Parthian": range(68416, 68448), + "Inscriptional Pahlavi": range(68448, 68480), + "Psalter Pahlavi": range(68480, 68528), + "Old Turkic": range(68608, 68688), + "Old Hungarian": range(68736, 68864), + "Hanifi Rohingya": range(68864, 68928), + "Rumi Numeral Symbols": range(69216, 69248), + "Yezidi": range(69248, 69312), + "Arabic Extended-C": range(69312, 69376), + "Old Sogdian": range(69376, 69424), + "Sogdian": range(69424, 69488), + "Old Uyghur": range(69488, 69552), + "Chorasmian": range(69552, 69600), + "Elymaic": range(69600, 69632), + "Brahmi": range(69632, 69760), + "Kaithi": range(69760, 69840), + "Sora Sompeng": range(69840, 69888), + "Chakma": range(69888, 69968), + "Mahajani": range(69968, 70016), + "Sharada": range(70016, 70112), + "Sinhala Archaic Numbers": range(70112, 70144), + "Khojki": range(70144, 70224), + "Multani": range(70272, 70320), + "Khudawadi": range(70320, 70400), + "Grantha": range(70400, 70528), + "Newa": range(70656, 70784), + "Tirhuta": range(70784, 70880), + "Siddham": range(71040, 71168), + "Modi": range(71168, 71264), + "Mongolian Supplement": range(71264, 71296), + "Takri": range(71296, 71376), + "Ahom": range(71424, 71504), + "Dogra": range(71680, 71760), + "Warang Citi": range(71840, 71936), + "Dives Akuru": range(71936, 72032), + "Nandinagari": range(72096, 72192), + "Zanabazar Square": range(72192, 72272), + "Soyombo": range(72272, 72368), + "Unified Canadian Aboriginal Syllabics Extended-A": range(72368, 72384), + "Pau Cin Hau": range(72384, 72448), + "Devanagari Extended-A": range(72448, 72544), + "Bhaiksuki": range(72704, 72816), + "Marchen": range(72816, 72896), + "Masaram Gondi": range(72960, 73056), + "Gunjala Gondi": range(73056, 73136), + "Makasar": range(73440, 73472), + "Kawi": range(73472, 73568), + "Lisu Supplement": range(73648, 73664), + "Tamil Supplement": range(73664, 73728), + "Cuneiform": range(73728, 74752), + "Cuneiform Numbers and Punctuation": range(74752, 74880), + "Early Dynastic Cuneiform": range(74880, 75088), + "Cypro-Minoan": range(77712, 77824), + "Egyptian Hieroglyphs": range(77824, 78896), + "Egyptian Hieroglyph Format Controls": range(78896, 78944), + "Anatolian Hieroglyphs": range(82944, 83584), + "Bamum Supplement": range(92160, 92736), + "Mro": range(92736, 92784), + "Tangsa": range(92784, 92880), + "Bassa Vah": range(92880, 92928), + "Pahawh Hmong": range(92928, 93072), + "Medefaidrin": range(93760, 93856), + "Miao": range(93952, 94112), + "Ideographic Symbols and Punctuation": range(94176, 94208), + "Tangut": range(94208, 100352), + "Tangut Components": range(100352, 101120), + "Khitan Small Script": range(101120, 101632), + "Tangut Supplement": range(101632, 101760), + "Kana Extended-B": range(110576, 110592), + "Kana Supplement": range(110592, 110848), + "Kana Extended-A": range(110848, 110896), + "Small Kana Extension": range(110896, 110960), + "Nushu": range(110960, 111360), + "Duployan": range(113664, 113824), + "Shorthand Format Controls": range(113824, 113840), + "Znamenny Musical Notation": range(118528, 118736), + "Byzantine Musical Symbols": range(118784, 119040), + "Musical Symbols": range(119040, 119296), + "Ancient Greek Musical Notation": range(119296, 119376), + "Kaktovik Numerals": range(119488, 119520), + "Mayan Numerals": range(119520, 119552), + "Tai Xuan Jing Symbols": range(119552, 119648), + "Counting Rod Numerals": range(119648, 119680), + "Mathematical Alphanumeric Symbols": range(119808, 120832), + "Sutton SignWriting": range(120832, 121520), + "Latin Extended-G": range(122624, 122880), + "Glagolitic Supplement": range(122880, 122928), + "Cyrillic Extended-D": range(122928, 123024), + "Nyiakeng Puachue Hmong": range(123136, 123216), + "Toto": range(123536, 123584), + "Wancho": range(123584, 123648), + "Nag Mundari": range(124112, 124160), + "Ethiopic Extended-B": range(124896, 124928), + "Mende Kikakui": range(124928, 125152), + "Adlam": range(125184, 125280), + "Indic Siyaq Numbers": range(126064, 126144), + "Ottoman Siyaq Numbers": range(126208, 126288), + "Arabic Mathematical Alphabetic Symbols": range(126464, 126720), + "Mahjong Tiles": range(126976, 127024), + "Domino Tiles": range(127024, 127136), + "Playing Cards": range(127136, 127232), + "Enclosed Alphanumeric Supplement": range(127232, 127488), + "Enclosed Ideographic Supplement": range(127488, 127744), + "Miscellaneous Symbols and Pictographs": range(127744, 128512), + "Emoticons range(Emoji)": range(128512, 128592), + "Ornamental Dingbats": range(128592, 128640), + "Transport and Map Symbols": range(128640, 128768), + "Alchemical Symbols": range(128768, 128896), + "Geometric Shapes Extended": range(128896, 129024), + "Supplemental Arrows-C": range(129024, 129280), + "Supplemental Symbols and Pictographs": range(129280, 129536), + "Chess Symbols": range(129536, 129648), + "Symbols and Pictographs Extended-A": range(129648, 129792), + "Symbols for Legacy Computing": range(129792, 130048), + "CJK Unified Ideographs Extension B": range(131072, 173792), + "CJK Unified Ideographs Extension C": range(173824, 177984), + "CJK Unified Ideographs Extension D": range(177984, 178208), + "CJK Unified Ideographs Extension E": range(178208, 183984), + "CJK Unified Ideographs Extension F": range(183984, 191472), + "CJK Compatibility Ideographs Supplement": range(194560, 195104), + "CJK Unified Ideographs Extension G": range(196608, 201552), + "CJK Unified Ideographs Extension H": range(201552, 205744), + "Tags": range(917504, 917632), + "Variation Selectors Supplement": range(917760, 918000), + "Supplementary Private Use Area-A": range(983040, 1048576), + "Supplementary Private Use Area-B": range(1048576, 1114112), +} + + +UNICODE_SECONDARY_RANGE_KEYWORD: list[str] = [ + "Supplement", + "Extended", + "Extensions", + "Modifier", + "Marks", + "Punctuation", + "Symbols", + "Forms", + "Operators", + "Miscellaneous", + "Drawing", + "Block", + "Shapes", + "Supplemental", + "Tags", +] + +RE_POSSIBLE_ENCODING_INDICATION = re_compile( + r"(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)", + IGNORECASE, +) + +IANA_NO_ALIASES = [ + "cp720", + "cp737", + "cp856", + "cp874", + "cp875", + "cp1006", + "koi8_r", + "koi8_t", + "koi8_u", +] + +IANA_SUPPORTED: list[str] = sorted( + filter( + lambda x: x.endswith("_codec") is False + and x not in {"rot_13", "tactis", "mbcs"}, + list(set(aliases.values())) + IANA_NO_ALIASES, + ) +) + +IANA_SUPPORTED_COUNT: int = len(IANA_SUPPORTED) + +# pre-computed code page that are similar using the function cp_similarity. +IANA_SUPPORTED_SIMILAR: dict[str, list[str]] = { + "cp037": ["cp1026", "cp1140", "cp273", "cp500"], + "cp1026": ["cp037", "cp1140", "cp273", "cp500"], + "cp1125": ["cp866"], + "cp1140": ["cp037", "cp1026", "cp273", "cp500"], + "cp1250": ["iso8859_2"], + "cp1251": ["kz1048", "ptcp154"], + "cp1252": ["iso8859_15", "iso8859_9", "latin_1"], + "cp1253": ["iso8859_7"], + "cp1254": ["iso8859_15", "iso8859_9", "latin_1"], + "cp1257": ["iso8859_13"], + "cp273": ["cp037", "cp1026", "cp1140", "cp500"], + "cp437": ["cp850", "cp858", "cp860", "cp861", "cp862", "cp863", "cp865"], + "cp500": ["cp037", "cp1026", "cp1140", "cp273"], + "cp850": ["cp437", "cp857", "cp858", "cp865"], + "cp857": ["cp850", "cp858", "cp865"], + "cp858": ["cp437", "cp850", "cp857", "cp865"], + "cp860": ["cp437", "cp861", "cp862", "cp863", "cp865"], + "cp861": ["cp437", "cp860", "cp862", "cp863", "cp865"], + "cp862": ["cp437", "cp860", "cp861", "cp863", "cp865"], + "cp863": ["cp437", "cp860", "cp861", "cp862", "cp865"], + "cp865": ["cp437", "cp850", "cp857", "cp858", "cp860", "cp861", "cp862", "cp863"], + "cp866": ["cp1125"], + "iso8859_10": ["iso8859_14", "iso8859_15", "iso8859_4", "iso8859_9", "latin_1"], + "iso8859_11": ["tis_620"], + "iso8859_13": ["cp1257"], + "iso8859_14": [ + "iso8859_10", + "iso8859_15", + "iso8859_16", + "iso8859_3", + "iso8859_9", + "latin_1", + ], + "iso8859_15": [ + "cp1252", + "cp1254", + "iso8859_10", + "iso8859_14", + "iso8859_16", + "iso8859_3", + "iso8859_9", + "latin_1", + ], + "iso8859_16": [ + "iso8859_14", + "iso8859_15", + "iso8859_2", + "iso8859_3", + "iso8859_9", + "latin_1", + ], + "iso8859_2": ["cp1250", "iso8859_16", "iso8859_4"], + "iso8859_3": ["iso8859_14", "iso8859_15", "iso8859_16", "iso8859_9", "latin_1"], + "iso8859_4": ["iso8859_10", "iso8859_2", "iso8859_9", "latin_1"], + "iso8859_7": ["cp1253"], + "iso8859_9": [ + "cp1252", + "cp1254", + "cp1258", + "iso8859_10", + "iso8859_14", + "iso8859_15", + "iso8859_16", + "iso8859_3", + "iso8859_4", + "latin_1", + ], + "kz1048": ["cp1251", "ptcp154"], + "latin_1": [ + "cp1252", + "cp1254", + "cp1258", + "iso8859_10", + "iso8859_14", + "iso8859_15", + "iso8859_16", + "iso8859_3", + "iso8859_4", + "iso8859_9", + ], + "mac_iceland": ["mac_roman", "mac_turkish"], + "mac_roman": ["mac_iceland", "mac_turkish"], + "mac_turkish": ["mac_iceland", "mac_roman"], + "ptcp154": ["cp1251", "kz1048"], + "tis_620": ["iso8859_11"], +} + + +CHARDET_CORRESPONDENCE: dict[str, str] = { + "iso2022_kr": "ISO-2022-KR", + "iso2022_jp": "ISO-2022-JP", + "euc_kr": "EUC-KR", + "tis_620": "TIS-620", + "utf_32": "UTF-32", + "euc_jp": "EUC-JP", + "koi8_r": "KOI8-R", + "iso8859_1": "ISO-8859-1", + "iso8859_2": "ISO-8859-2", + "iso8859_5": "ISO-8859-5", + "iso8859_6": "ISO-8859-6", + "iso8859_7": "ISO-8859-7", + "iso8859_8": "ISO-8859-8", + "utf_16": "UTF-16", + "cp855": "IBM855", + "mac_cyrillic": "MacCyrillic", + "gb2312": "GB2312", + "gb18030": "GB18030", + "cp932": "CP932", + "cp866": "IBM866", + "utf_8": "utf-8", + "utf_8_sig": "UTF-8-SIG", + "shift_jis": "SHIFT_JIS", + "big5": "Big5", + "cp1250": "windows-1250", + "cp1251": "windows-1251", + "cp1252": "Windows-1252", + "cp1253": "windows-1253", + "cp1255": "windows-1255", + "cp1256": "windows-1256", + "cp1254": "Windows-1254", + "cp949": "CP949", +} + + +COMMON_SAFE_ASCII_CHARACTERS: set[str] = { + "<", + ">", + "=", + ":", + "/", + "&", + ";", + "{", + "}", + "[", + "]", + ",", + "|", + '"', + "-", + "(", + ")", +} + + +KO_NAMES: set[str] = {"johab", "cp949", "euc_kr"} +ZH_NAMES: set[str] = {"big5", "cp950", "big5hkscs", "hz"} + +# Logging LEVEL below DEBUG +TRACE: int = 5 + + +# Language label that contain the em dash "—" +# character are to be considered alternative seq to origin +FREQUENCIES: dict[str, list[str]] = { + "English": [ + "e", + "a", + "t", + "i", + "o", + "n", + "s", + "r", + "h", + "l", + "d", + "c", + "u", + "m", + "f", + "p", + "g", + "w", + "y", + "b", + "v", + "k", + "x", + "j", + "z", + "q", + ], + "English—": [ + "e", + "a", + "t", + "i", + "o", + "n", + "s", + "r", + "h", + "l", + "d", + "c", + "m", + "u", + "f", + "p", + "g", + "w", + "b", + "y", + "v", + "k", + "j", + "x", + "z", + "q", + ], + "German": [ + "e", + "n", + "i", + "r", + "s", + "t", + "a", + "d", + "h", + "u", + "l", + "g", + "o", + "c", + "m", + "b", + "f", + "k", + "w", + "z", + "p", + "v", + "ü", + "ä", + "ö", + "j", + ], + "French": [ + "e", + "a", + "s", + "n", + "i", + "t", + "r", + "l", + "u", + "o", + "d", + "c", + "p", + "m", + "é", + "v", + "g", + "f", + "b", + "h", + "q", + "à", + "x", + "è", + "y", + "j", + ], + "Dutch": [ + "e", + "n", + "a", + "i", + "r", + "t", + "o", + "d", + "s", + "l", + "g", + "h", + "v", + "m", + "u", + "k", + "c", + "p", + "b", + "w", + "j", + "z", + "f", + "y", + "x", + "ë", + ], + "Italian": [ + "e", + "i", + "a", + "o", + "n", + "l", + "t", + "r", + "s", + "c", + "d", + "u", + "p", + "m", + "g", + "v", + "f", + "b", + "z", + "h", + "q", + "è", + "à", + "k", + "y", + "ò", + ], + "Polish": [ + "a", + "i", + "o", + "e", + "n", + "r", + "z", + "w", + "s", + "c", + "t", + "k", + "y", + "d", + "p", + "m", + "u", + "l", + "j", + "ł", + "g", + "b", + "h", + "ą", + "ę", + "ó", + ], + "Spanish": [ + "e", + "a", + "o", + "n", + "s", + "r", + "i", + "l", + "d", + "t", + "c", + "u", + "m", + "p", + "b", + "g", + "v", + "f", + "y", + "ó", + "h", + "q", + "í", + "j", + "z", + "á", + ], + "Russian": [ + "о", + "а", + "е", + "и", + "н", + "с", + "т", + "р", + "в", + "л", + "к", + "м", + "д", + "п", + "у", + "г", + "я", + "ы", + "з", + "б", + "й", + "ь", + "ч", + "х", + "ж", + "ц", + ], + # Jap-Kanji + "Japanese": [ + "人", + "一", + "大", + "亅", + "丁", + "丨", + "竹", + "笑", + "口", + "日", + "今", + "二", + "彳", + "行", + "十", + "土", + "丶", + "寸", + "寺", + "時", + "乙", + "丿", + "乂", + "气", + "気", + "冂", + "巾", + "亠", + "市", + "目", + "儿", + "見", + "八", + "小", + "凵", + "県", + "月", + "彐", + "門", + "間", + "木", + "東", + "山", + "出", + "本", + "中", + "刀", + "分", + "耳", + "又", + "取", + "最", + "言", + "田", + "心", + "思", + "刂", + "前", + "京", + "尹", + "事", + "生", + "厶", + "云", + "会", + "未", + "来", + "白", + "冫", + "楽", + "灬", + "馬", + "尸", + "尺", + "駅", + "明", + "耂", + "者", + "了", + "阝", + "都", + "高", + "卜", + "占", + "厂", + "广", + "店", + "子", + "申", + "奄", + "亻", + "俺", + "上", + "方", + "冖", + "学", + "衣", + "艮", + "食", + "自", + ], + # Jap-Katakana + "Japanese—": [ + "ー", + "ン", + "ス", + "・", + "ル", + "ト", + "リ", + "イ", + "ア", + "ラ", + "ッ", + "ク", + "ド", + "シ", + "レ", + "ジ", + "タ", + "フ", + "ロ", + "カ", + "テ", + "マ", + "ィ", + "グ", + "バ", + "ム", + "プ", + "オ", + "コ", + "デ", + "ニ", + "ウ", + "メ", + "サ", + "ビ", + "ナ", + "ブ", + "ャ", + "エ", + "ュ", + "チ", + "キ", + "ズ", + "ダ", + "パ", + "ミ", + "ェ", + "ョ", + "ハ", + "セ", + "ベ", + "ガ", + "モ", + "ツ", + "ネ", + "ボ", + "ソ", + "ノ", + "ァ", + "ヴ", + "ワ", + "ポ", + "ペ", + "ピ", + "ケ", + "ゴ", + "ギ", + "ザ", + "ホ", + "ゲ", + "ォ", + "ヤ", + "ヒ", + "ユ", + "ヨ", + "ヘ", + "ゼ", + "ヌ", + "ゥ", + "ゾ", + "ヶ", + "ヂ", + "ヲ", + "ヅ", + "ヵ", + "ヱ", + "ヰ", + "ヮ", + "ヽ", + "゠", + "ヾ", + "ヷ", + "ヿ", + "ヸ", + "ヹ", + "ヺ", + ], + # Jap-Hiragana + "Japanese——": [ + "の", + "に", + "る", + "た", + "と", + "は", + "し", + "い", + "を", + "で", + "て", + "が", + "な", + "れ", + "か", + "ら", + "さ", + "っ", + "り", + "す", + "あ", + "も", + "こ", + "ま", + "う", + "く", + "よ", + "き", + "ん", + "め", + "お", + "け", + "そ", + "つ", + "だ", + "や", + "え", + "ど", + "わ", + "ち", + "み", + "せ", + "じ", + "ば", + "へ", + "び", + "ず", + "ろ", + "ほ", + "げ", + "む", + "べ", + "ひ", + "ょ", + "ゆ", + "ぶ", + "ご", + "ゃ", + "ね", + "ふ", + "ぐ", + "ぎ", + "ぼ", + "ゅ", + "づ", + "ざ", + "ぞ", + "ぬ", + "ぜ", + "ぱ", + "ぽ", + "ぷ", + "ぴ", + "ぃ", + "ぁ", + "ぇ", + "ぺ", + "ゞ", + "ぢ", + "ぉ", + "ぅ", + "ゐ", + "ゝ", + "ゑ", + "゛", + "゜", + "ゎ", + "ゔ", + "゚", + "ゟ", + "゙", + "ゕ", + "ゖ", + ], + "Portuguese": [ + "a", + "e", + "o", + "s", + "i", + "r", + "d", + "n", + "t", + "m", + "u", + "c", + "l", + "p", + "g", + "v", + "b", + "f", + "h", + "ã", + "q", + "é", + "ç", + "á", + "z", + "í", + ], + "Swedish": [ + "e", + "a", + "n", + "r", + "t", + "s", + "i", + "l", + "d", + "o", + "m", + "k", + "g", + "v", + "h", + "f", + "u", + "p", + "ä", + "c", + "b", + "ö", + "å", + "y", + "j", + "x", + ], + "Chinese": [ + "的", + "一", + "是", + "不", + "了", + "在", + "人", + "有", + "我", + "他", + "这", + "个", + "们", + "中", + "来", + "上", + "大", + "为", + "和", + "国", + "地", + "到", + "以", + "说", + "时", + "要", + "就", + "出", + "会", + "可", + "也", + "你", + "对", + "生", + "能", + "而", + "子", + "那", + "得", + "于", + "着", + "下", + "自", + "之", + "年", + "过", + "发", + "后", + "作", + "里", + "用", + "道", + "行", + "所", + "然", + "家", + "种", + "事", + "成", + "方", + "多", + "经", + "么", + "去", + "法", + "学", + "如", + "都", + "同", + "现", + "当", + "没", + "动", + "面", + "起", + "看", + "定", + "天", + "分", + "还", + "进", + "好", + "小", + "部", + "其", + "些", + "主", + "样", + "理", + "心", + "她", + "本", + "前", + "开", + "但", + "因", + "只", + "从", + "想", + "实", + ], + "Ukrainian": [ + "о", + "а", + "н", + "і", + "и", + "р", + "в", + "т", + "е", + "с", + "к", + "л", + "у", + "д", + "м", + "п", + "з", + "я", + "ь", + "б", + "г", + "й", + "ч", + "х", + "ц", + "ї", + ], + "Norwegian": [ + "e", + "r", + "n", + "t", + "a", + "s", + "i", + "o", + "l", + "d", + "g", + "k", + "m", + "v", + "f", + "p", + "u", + "b", + "h", + "å", + "y", + "j", + "ø", + "c", + "æ", + "w", + ], + "Finnish": [ + "a", + "i", + "n", + "t", + "e", + "s", + "l", + "o", + "u", + "k", + "ä", + "m", + "r", + "v", + "j", + "h", + "p", + "y", + "d", + "ö", + "g", + "c", + "b", + "f", + "w", + "z", + ], + "Vietnamese": [ + "n", + "h", + "t", + "i", + "c", + "g", + "a", + "o", + "u", + "m", + "l", + "r", + "à", + "đ", + "s", + "e", + "v", + "p", + "b", + "y", + "ư", + "d", + "á", + "k", + "ộ", + "ế", + ], + "Czech": [ + "o", + "e", + "a", + "n", + "t", + "s", + "i", + "l", + "v", + "r", + "k", + "d", + "u", + "m", + "p", + "í", + "c", + "h", + "z", + "á", + "y", + "j", + "b", + "ě", + "é", + "ř", + ], + "Hungarian": [ + "e", + "a", + "t", + "l", + "s", + "n", + "k", + "r", + "i", + "o", + "z", + "á", + "é", + "g", + "m", + "b", + "y", + "v", + "d", + "h", + "u", + "p", + "j", + "ö", + "f", + "c", + ], + "Korean": [ + "이", + "다", + "에", + "의", + "는", + "로", + "하", + "을", + "가", + "고", + "지", + "서", + "한", + "은", + "기", + "으", + "년", + "대", + "사", + "시", + "를", + "리", + "도", + "인", + "스", + "일", + ], + "Indonesian": [ + "a", + "n", + "e", + "i", + "r", + "t", + "u", + "s", + "d", + "k", + "m", + "l", + "g", + "p", + "b", + "o", + "h", + "y", + "j", + "c", + "w", + "f", + "v", + "z", + "x", + "q", + ], + "Turkish": [ + "a", + "e", + "i", + "n", + "r", + "l", + "ı", + "k", + "d", + "t", + "s", + "m", + "y", + "u", + "o", + "b", + "ü", + "ş", + "v", + "g", + "z", + "h", + "c", + "p", + "ç", + "ğ", + ], + "Romanian": [ + "e", + "i", + "a", + "r", + "n", + "t", + "u", + "l", + "o", + "c", + "s", + "d", + "p", + "m", + "ă", + "f", + "v", + "î", + "g", + "b", + "ș", + "ț", + "z", + "h", + "â", + "j", + ], + "Farsi": [ + "ا", + "ی", + "ر", + "د", + "ن", + "ه", + "و", + "م", + "ت", + "ب", + "س", + "ل", + "ک", + "ش", + "ز", + "ف", + "گ", + "ع", + "خ", + "ق", + "ج", + "آ", + "پ", + "ح", + "ط", + "ص", + ], + "Arabic": [ + "ا", + "ل", + "ي", + "م", + "و", + "ن", + "ر", + "ت", + "ب", + "ة", + "ع", + "د", + "س", + "ف", + "ه", + "ك", + "ق", + "أ", + "ح", + "ج", + "ش", + "ط", + "ص", + "ى", + "خ", + "إ", + ], + "Danish": [ + "e", + "r", + "n", + "t", + "a", + "i", + "s", + "d", + "l", + "o", + "g", + "m", + "k", + "f", + "v", + "u", + "b", + "h", + "p", + "å", + "y", + "ø", + "æ", + "c", + "j", + "w", + ], + "Serbian": [ + "а", + "и", + "о", + "е", + "н", + "р", + "с", + "у", + "т", + "к", + "ј", + "в", + "д", + "м", + "п", + "л", + "г", + "з", + "б", + "a", + "i", + "e", + "o", + "n", + "ц", + "ш", + ], + "Lithuanian": [ + "i", + "a", + "s", + "o", + "r", + "e", + "t", + "n", + "u", + "k", + "m", + "l", + "p", + "v", + "d", + "j", + "g", + "ė", + "b", + "y", + "ų", + "š", + "ž", + "c", + "ą", + "į", + ], + "Slovene": [ + "e", + "a", + "i", + "o", + "n", + "r", + "s", + "l", + "t", + "j", + "v", + "k", + "d", + "p", + "m", + "u", + "z", + "b", + "g", + "h", + "č", + "c", + "š", + "ž", + "f", + "y", + ], + "Slovak": [ + "o", + "a", + "e", + "n", + "i", + "r", + "v", + "t", + "s", + "l", + "k", + "d", + "m", + "p", + "u", + "c", + "h", + "j", + "b", + "z", + "á", + "y", + "ý", + "í", + "č", + "é", + ], + "Hebrew": [ + "י", + "ו", + "ה", + "ל", + "ר", + "ב", + "ת", + "מ", + "א", + "ש", + "נ", + "ע", + "ם", + "ד", + "ק", + "ח", + "פ", + "ס", + "כ", + "ג", + "ט", + "צ", + "ן", + "ז", + "ך", + ], + "Bulgarian": [ + "а", + "и", + "о", + "е", + "н", + "т", + "р", + "с", + "в", + "л", + "к", + "д", + "п", + "м", + "з", + "г", + "я", + "ъ", + "у", + "б", + "ч", + "ц", + "й", + "ж", + "щ", + "х", + ], + "Croatian": [ + "a", + "i", + "o", + "e", + "n", + "r", + "j", + "s", + "t", + "u", + "k", + "l", + "v", + "d", + "m", + "p", + "g", + "z", + "b", + "c", + "č", + "h", + "š", + "ž", + "ć", + "f", + ], + "Hindi": [ + "क", + "र", + "स", + "न", + "त", + "म", + "ह", + "प", + "य", + "ल", + "व", + "ज", + "द", + "ग", + "ब", + "श", + "ट", + "अ", + "ए", + "थ", + "भ", + "ड", + "च", + "ध", + "ष", + "इ", + ], + "Estonian": [ + "a", + "i", + "e", + "s", + "t", + "l", + "u", + "n", + "o", + "k", + "r", + "d", + "m", + "v", + "g", + "p", + "j", + "h", + "ä", + "b", + "õ", + "ü", + "f", + "c", + "ö", + "y", + ], + "Thai": [ + "า", + "น", + "ร", + "อ", + "ก", + "เ", + "ง", + "ม", + "ย", + "ล", + "ว", + "ด", + "ท", + "ส", + "ต", + "ะ", + "ป", + "บ", + "ค", + "ห", + "แ", + "จ", + "พ", + "ช", + "ข", + "ใ", + ], + "Greek": [ + "α", + "τ", + "ο", + "ι", + "ε", + "ν", + "ρ", + "σ", + "κ", + "η", + "π", + "ς", + "υ", + "μ", + "λ", + "ί", + "ό", + "ά", + "γ", + "έ", + "δ", + "ή", + "ω", + "χ", + "θ", + "ύ", + ], + "Tamil": [ + "க", + "த", + "ப", + "ட", + "ர", + "ம", + "ல", + "ன", + "வ", + "ற", + "ய", + "ள", + "ச", + "ந", + "இ", + "ண", + "அ", + "ஆ", + "ழ", + "ங", + "எ", + "உ", + "ஒ", + "ஸ", + ], + "Kazakh": [ + "а", + "ы", + "е", + "н", + "т", + "р", + "л", + "і", + "д", + "с", + "м", + "қ", + "к", + "о", + "б", + "и", + "у", + "ғ", + "ж", + "ң", + "з", + "ш", + "й", + "п", + "г", + "ө", + ], +} + +LANGUAGE_SUPPORTED_COUNT: int = len(FREQUENCIES) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/legacy.py b/tools/retro-library/charset_normalizer/charset_normalizer/legacy.py new file mode 100644 index 000000000..cfb876a19 --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/legacy.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from warnings import warn + +from .api import from_bytes +from .constant import CHARDET_CORRESPONDENCE + +# TODO: remove this check when dropping Python 3.7 support +if TYPE_CHECKING: + from typing_extensions import TypedDict + + class ResultDict(TypedDict): + encoding: str | None + language: str + confidence: float | None + + +def detect( + byte_str: bytes, should_rename_legacy: bool = False, **kwargs: Any +) -> ResultDict: + """ + chardet legacy method + Detect the encoding of the given byte string. It should be mostly backward-compatible. + Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it) + This function is deprecated and should be used to migrate your project easily, consult the documentation for + further information. Not planned for removal. + + :param byte_str: The byte sequence to examine. + :param should_rename_legacy: Should we rename legacy encodings + to their more modern equivalents? + """ + if len(kwargs): + warn( + f"charset-normalizer disregard arguments '{','.join(list(kwargs.keys()))}' in legacy function detect()" + ) + + if not isinstance(byte_str, (bytearray, bytes)): + raise TypeError( # pragma: nocover + "Expected object of type bytes or bytearray, got: " + "{}".format(type(byte_str)) + ) + + if isinstance(byte_str, bytearray): + byte_str = bytes(byte_str) + + r = from_bytes(byte_str).best() + + encoding = r.encoding if r is not None else None + language = r.language if r is not None and r.language != "Unknown" else "" + confidence = 1.0 - r.chaos if r is not None else None + + # Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process + # but chardet does return 'utf-8-sig' and it is a valid codec name. + if r is not None and encoding == "utf_8" and r.bom: + encoding += "_sig" + + if should_rename_legacy is False and encoding in CHARDET_CORRESPONDENCE: + encoding = CHARDET_CORRESPONDENCE[encoding] + + return { + "encoding": encoding, + "language": language, + "confidence": confidence, + } diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/md.py b/tools/retro-library/charset_normalizer/charset_normalizer/md.py new file mode 100644 index 000000000..d177db01f --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/md.py @@ -0,0 +1,629 @@ +from __future__ import annotations + +from functools import lru_cache +from logging import getLogger + +from .constant import ( + COMMON_SAFE_ASCII_CHARACTERS, + TRACE, + UNICODE_SECONDARY_RANGE_KEYWORD, +) +from .utils import ( + is_accentuated, + is_arabic, + is_arabic_isolated_form, + is_case_variable, + is_cjk, + is_emoticon, + is_hangul, + is_hiragana, + is_katakana, + is_latin, + is_punctuation, + is_separator, + is_symbol, + is_thai, + is_unprintable, + remove_accent, + unicode_range, +) + + +class MessDetectorPlugin: + """ + Base abstract class used for mess detection plugins. + All detectors MUST extend and implement given methods. + """ + + def eligible(self, character: str) -> bool: + """ + Determine if given character should be fed in. + """ + raise NotImplementedError # pragma: nocover + + def feed(self, character: str) -> None: + """ + The main routine to be executed upon character. + Insert the logic in witch the text would be considered chaotic. + """ + raise NotImplementedError # pragma: nocover + + def reset(self) -> None: # pragma: no cover + """ + Permit to reset the plugin to the initial state. + """ + raise NotImplementedError + + @property + def ratio(self) -> float: + """ + Compute the chaos ratio based on what your feed() has seen. + Must NOT be lower than 0.; No restriction gt 0. + """ + raise NotImplementedError # pragma: nocover + + +class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin): + def __init__(self) -> None: + self._punctuation_count: int = 0 + self._symbol_count: int = 0 + self._character_count: int = 0 + + self._last_printable_char: str | None = None + self._frenzy_symbol_in_word: bool = False + + def eligible(self, character: str) -> bool: + return character.isprintable() + + def feed(self, character: str) -> None: + self._character_count += 1 + + if ( + character != self._last_printable_char + and character not in COMMON_SAFE_ASCII_CHARACTERS + ): + if is_punctuation(character): + self._punctuation_count += 1 + elif ( + character.isdigit() is False + and is_symbol(character) + and is_emoticon(character) is False + ): + self._symbol_count += 2 + + self._last_printable_char = character + + def reset(self) -> None: # pragma: no cover + self._punctuation_count = 0 + self._character_count = 0 + self._symbol_count = 0 + + @property + def ratio(self) -> float: + if self._character_count == 0: + return 0.0 + + ratio_of_punctuation: float = ( + self._punctuation_count + self._symbol_count + ) / self._character_count + + return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0 + + +class TooManyAccentuatedPlugin(MessDetectorPlugin): + def __init__(self) -> None: + self._character_count: int = 0 + self._accentuated_count: int = 0 + + def eligible(self, character: str) -> bool: + return character.isalpha() + + def feed(self, character: str) -> None: + self._character_count += 1 + + if is_accentuated(character): + self._accentuated_count += 1 + + def reset(self) -> None: # pragma: no cover + self._character_count = 0 + self._accentuated_count = 0 + + @property + def ratio(self) -> float: + if self._character_count < 8: + return 0.0 + + ratio_of_accentuation: float = self._accentuated_count / self._character_count + return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0 + + +class UnprintablePlugin(MessDetectorPlugin): + def __init__(self) -> None: + self._unprintable_count: int = 0 + self._character_count: int = 0 + + def eligible(self, character: str) -> bool: + return True + + def feed(self, character: str) -> None: + if is_unprintable(character): + self._unprintable_count += 1 + self._character_count += 1 + + def reset(self) -> None: # pragma: no cover + self._unprintable_count = 0 + + @property + def ratio(self) -> float: + if self._character_count == 0: + return 0.0 + + return (self._unprintable_count * 8) / self._character_count + + +class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin): + def __init__(self) -> None: + self._successive_count: int = 0 + self._character_count: int = 0 + + self._last_latin_character: str | None = None + + def eligible(self, character: str) -> bool: + return character.isalpha() and is_latin(character) + + def feed(self, character: str) -> None: + self._character_count += 1 + if ( + self._last_latin_character is not None + and is_accentuated(character) + and is_accentuated(self._last_latin_character) + ): + if character.isupper() and self._last_latin_character.isupper(): + self._successive_count += 1 + # Worse if its the same char duplicated with different accent. + if remove_accent(character) == remove_accent(self._last_latin_character): + self._successive_count += 1 + self._last_latin_character = character + + def reset(self) -> None: # pragma: no cover + self._successive_count = 0 + self._character_count = 0 + self._last_latin_character = None + + @property + def ratio(self) -> float: + if self._character_count == 0: + return 0.0 + + return (self._successive_count * 2) / self._character_count + + +class SuspiciousRange(MessDetectorPlugin): + def __init__(self) -> None: + self._suspicious_successive_range_count: int = 0 + self._character_count: int = 0 + self._last_printable_seen: str | None = None + + def eligible(self, character: str) -> bool: + return character.isprintable() + + def feed(self, character: str) -> None: + self._character_count += 1 + + if ( + character.isspace() + or is_punctuation(character) + or character in COMMON_SAFE_ASCII_CHARACTERS + ): + self._last_printable_seen = None + return + + if self._last_printable_seen is None: + self._last_printable_seen = character + return + + unicode_range_a: str | None = unicode_range(self._last_printable_seen) + unicode_range_b: str | None = unicode_range(character) + + if is_suspiciously_successive_range(unicode_range_a, unicode_range_b): + self._suspicious_successive_range_count += 1 + + self._last_printable_seen = character + + def reset(self) -> None: # pragma: no cover + self._character_count = 0 + self._suspicious_successive_range_count = 0 + self._last_printable_seen = None + + @property + def ratio(self) -> float: + if self._character_count <= 13: + return 0.0 + + ratio_of_suspicious_range_usage: float = ( + self._suspicious_successive_range_count * 2 + ) / self._character_count + + return ratio_of_suspicious_range_usage + + +class SuperWeirdWordPlugin(MessDetectorPlugin): + def __init__(self) -> None: + self._word_count: int = 0 + self._bad_word_count: int = 0 + self._foreign_long_count: int = 0 + + self._is_current_word_bad: bool = False + self._foreign_long_watch: bool = False + + self._character_count: int = 0 + self._bad_character_count: int = 0 + + self._buffer: str = "" + self._buffer_accent_count: int = 0 + self._buffer_glyph_count: int = 0 + + def eligible(self, character: str) -> bool: + return True + + def feed(self, character: str) -> None: + if character.isalpha(): + self._buffer += character + if is_accentuated(character): + self._buffer_accent_count += 1 + if ( + self._foreign_long_watch is False + and (is_latin(character) is False or is_accentuated(character)) + and is_cjk(character) is False + and is_hangul(character) is False + and is_katakana(character) is False + and is_hiragana(character) is False + and is_thai(character) is False + ): + self._foreign_long_watch = True + if ( + is_cjk(character) + or is_hangul(character) + or is_katakana(character) + or is_hiragana(character) + or is_thai(character) + ): + self._buffer_glyph_count += 1 + return + if not self._buffer: + return + if ( + character.isspace() or is_punctuation(character) or is_separator(character) + ) and self._buffer: + self._word_count += 1 + buffer_length: int = len(self._buffer) + + self._character_count += buffer_length + + if buffer_length >= 4: + if self._buffer_accent_count / buffer_length >= 0.5: + self._is_current_word_bad = True + # Word/Buffer ending with an upper case accentuated letter are so rare, + # that we will consider them all as suspicious. Same weight as foreign_long suspicious. + elif ( + is_accentuated(self._buffer[-1]) + and self._buffer[-1].isupper() + and all(_.isupper() for _ in self._buffer) is False + ): + self._foreign_long_count += 1 + self._is_current_word_bad = True + elif self._buffer_glyph_count == 1: + self._is_current_word_bad = True + self._foreign_long_count += 1 + if buffer_length >= 24 and self._foreign_long_watch: + camel_case_dst = [ + i + for c, i in zip(self._buffer, range(0, buffer_length)) + if c.isupper() + ] + probable_camel_cased: bool = False + + if camel_case_dst and (len(camel_case_dst) / buffer_length <= 0.3): + probable_camel_cased = True + + if not probable_camel_cased: + self._foreign_long_count += 1 + self._is_current_word_bad = True + + if self._is_current_word_bad: + self._bad_word_count += 1 + self._bad_character_count += len(self._buffer) + self._is_current_word_bad = False + + self._foreign_long_watch = False + self._buffer = "" + self._buffer_accent_count = 0 + self._buffer_glyph_count = 0 + elif ( + character not in {"<", ">", "-", "=", "~", "|", "_"} + and character.isdigit() is False + and is_symbol(character) + ): + self._is_current_word_bad = True + self._buffer += character + + def reset(self) -> None: # pragma: no cover + self._buffer = "" + self._is_current_word_bad = False + self._foreign_long_watch = False + self._bad_word_count = 0 + self._word_count = 0 + self._character_count = 0 + self._bad_character_count = 0 + self._foreign_long_count = 0 + + @property + def ratio(self) -> float: + if self._word_count <= 10 and self._foreign_long_count == 0: + return 0.0 + + return self._bad_character_count / self._character_count + + +class CjkInvalidStopPlugin(MessDetectorPlugin): + """ + GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and + can be easily detected. Searching for the overuse of '丅' and '丄'. + """ + + def __init__(self) -> None: + self._wrong_stop_count: int = 0 + self._cjk_character_count: int = 0 + + def eligible(self, character: str) -> bool: + return True + + def feed(self, character: str) -> None: + if character in {"丅", "丄"}: + self._wrong_stop_count += 1 + return + if is_cjk(character): + self._cjk_character_count += 1 + + def reset(self) -> None: # pragma: no cover + self._wrong_stop_count = 0 + self._cjk_character_count = 0 + + @property + def ratio(self) -> float: + if self._cjk_character_count < 16: + return 0.0 + return self._wrong_stop_count / self._cjk_character_count + + +class ArchaicUpperLowerPlugin(MessDetectorPlugin): + def __init__(self) -> None: + self._buf: bool = False + + self._character_count_since_last_sep: int = 0 + + self._successive_upper_lower_count: int = 0 + self._successive_upper_lower_count_final: int = 0 + + self._character_count: int = 0 + + self._last_alpha_seen: str | None = None + self._current_ascii_only: bool = True + + def eligible(self, character: str) -> bool: + return True + + def feed(self, character: str) -> None: + is_concerned = character.isalpha() and is_case_variable(character) + chunk_sep = is_concerned is False + + if chunk_sep and self._character_count_since_last_sep > 0: + if ( + self._character_count_since_last_sep <= 64 + and character.isdigit() is False + and self._current_ascii_only is False + ): + self._successive_upper_lower_count_final += ( + self._successive_upper_lower_count + ) + + self._successive_upper_lower_count = 0 + self._character_count_since_last_sep = 0 + self._last_alpha_seen = None + self._buf = False + self._character_count += 1 + self._current_ascii_only = True + + return + + if self._current_ascii_only is True and character.isascii() is False: + self._current_ascii_only = False + + if self._last_alpha_seen is not None: + if (character.isupper() and self._last_alpha_seen.islower()) or ( + character.islower() and self._last_alpha_seen.isupper() + ): + if self._buf is True: + self._successive_upper_lower_count += 2 + self._buf = False + else: + self._buf = True + else: + self._buf = False + + self._character_count += 1 + self._character_count_since_last_sep += 1 + self._last_alpha_seen = character + + def reset(self) -> None: # pragma: no cover + self._character_count = 0 + self._character_count_since_last_sep = 0 + self._successive_upper_lower_count = 0 + self._successive_upper_lower_count_final = 0 + self._last_alpha_seen = None + self._buf = False + self._current_ascii_only = True + + @property + def ratio(self) -> float: + if self._character_count == 0: + return 0.0 + + return self._successive_upper_lower_count_final / self._character_count + + +class ArabicIsolatedFormPlugin(MessDetectorPlugin): + def __init__(self) -> None: + self._character_count: int = 0 + self._isolated_form_count: int = 0 + + def reset(self) -> None: # pragma: no cover + self._character_count = 0 + self._isolated_form_count = 0 + + def eligible(self, character: str) -> bool: + return is_arabic(character) + + def feed(self, character: str) -> None: + self._character_count += 1 + + if is_arabic_isolated_form(character): + self._isolated_form_count += 1 + + @property + def ratio(self) -> float: + if self._character_count < 8: + return 0.0 + + isolated_form_usage: float = self._isolated_form_count / self._character_count + + return isolated_form_usage + + +@lru_cache(maxsize=1024) +def is_suspiciously_successive_range( + unicode_range_a: str | None, unicode_range_b: str | None +) -> bool: + """ + Determine if two Unicode range seen next to each other can be considered as suspicious. + """ + if unicode_range_a is None or unicode_range_b is None: + return True + + if unicode_range_a == unicode_range_b: + return False + + if "Latin" in unicode_range_a and "Latin" in unicode_range_b: + return False + + if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b: + return False + + # Latin characters can be accompanied with a combining diacritical mark + # eg. Vietnamese. + if ("Latin" in unicode_range_a or "Latin" in unicode_range_b) and ( + "Combining" in unicode_range_a or "Combining" in unicode_range_b + ): + return False + + keywords_range_a, keywords_range_b = unicode_range_a.split( + " " + ), unicode_range_b.split(" ") + + for el in keywords_range_a: + if el in UNICODE_SECONDARY_RANGE_KEYWORD: + continue + if el in keywords_range_b: + return False + + # Japanese Exception + range_a_jp_chars, range_b_jp_chars = ( + unicode_range_a + in ( + "Hiragana", + "Katakana", + ), + unicode_range_b in ("Hiragana", "Katakana"), + ) + if (range_a_jp_chars or range_b_jp_chars) and ( + "CJK" in unicode_range_a or "CJK" in unicode_range_b + ): + return False + if range_a_jp_chars and range_b_jp_chars: + return False + + if "Hangul" in unicode_range_a or "Hangul" in unicode_range_b: + if "CJK" in unicode_range_a or "CJK" in unicode_range_b: + return False + if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin": + return False + + # Chinese/Japanese use dedicated range for punctuation and/or separators. + if ("CJK" in unicode_range_a or "CJK" in unicode_range_b) or ( + unicode_range_a in ["Katakana", "Hiragana"] + and unicode_range_b in ["Katakana", "Hiragana"] + ): + if "Punctuation" in unicode_range_a or "Punctuation" in unicode_range_b: + return False + if "Forms" in unicode_range_a or "Forms" in unicode_range_b: + return False + if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin": + return False + + return True + + +@lru_cache(maxsize=2048) +def mess_ratio( + decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False +) -> float: + """ + Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier. + """ + + detectors: list[MessDetectorPlugin] = [ + md_class() for md_class in MessDetectorPlugin.__subclasses__() + ] + + length: int = len(decoded_sequence) + 1 + + mean_mess_ratio: float = 0.0 + + if length < 512: + intermediary_mean_mess_ratio_calc: int = 32 + elif length <= 1024: + intermediary_mean_mess_ratio_calc = 64 + else: + intermediary_mean_mess_ratio_calc = 128 + + for character, index in zip(decoded_sequence + "\n", range(length)): + for detector in detectors: + if detector.eligible(character): + detector.feed(character) + + if ( + index > 0 and index % intermediary_mean_mess_ratio_calc == 0 + ) or index == length - 1: + mean_mess_ratio = sum(dt.ratio for dt in detectors) + + if mean_mess_ratio >= maximum_threshold: + break + + if debug: + logger = getLogger("charset_normalizer") + + logger.log( + TRACE, + "Mess-detector extended-analysis start. " + f"intermediary_mean_mess_ratio_calc={intermediary_mean_mess_ratio_calc} mean_mess_ratio={mean_mess_ratio} " + f"maximum_threshold={maximum_threshold}", + ) + + if len(decoded_sequence) > 16: + logger.log(TRACE, f"Starting with: {decoded_sequence[:16]}") + logger.log(TRACE, f"Ending with: {decoded_sequence[-16::]}") + + for dt in detectors: # pragma: nocover + logger.log(TRACE, f"{dt.__class__}: {dt.ratio}") + + return round(mean_mess_ratio, 3) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/models.py b/tools/retro-library/charset_normalizer/charset_normalizer/models.py new file mode 100644 index 000000000..09492a9cb --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/models.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +from encodings.aliases import aliases +from hashlib import sha256 +from json import dumps +from re import sub +from typing import Any, Iterator, List, Tuple + +from .constant import RE_POSSIBLE_ENCODING_INDICATION, TOO_BIG_SEQUENCE +from .utils import iana_name, is_multi_byte_encoding, unicode_range + + +class CharsetMatch: + def __init__( + self, + payload: bytes, + guessed_encoding: str, + mean_mess_ratio: float, + has_sig_or_bom: bool, + languages: CoherenceMatches, + decoded_payload: str | None = None, + preemptive_declaration: str | None = None, + ): + self._payload: bytes = payload + + self._encoding: str = guessed_encoding + self._mean_mess_ratio: float = mean_mess_ratio + self._languages: CoherenceMatches = languages + self._has_sig_or_bom: bool = has_sig_or_bom + self._unicode_ranges: list[str] | None = None + + self._leaves: list[CharsetMatch] = [] + self._mean_coherence_ratio: float = 0.0 + + self._output_payload: bytes | None = None + self._output_encoding: str | None = None + + self._string: str | None = decoded_payload + + self._preemptive_declaration: str | None = preemptive_declaration + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CharsetMatch): + if isinstance(other, str): + return iana_name(other) == self.encoding + return False + return self.encoding == other.encoding and self.fingerprint == other.fingerprint + + def __lt__(self, other: object) -> bool: + """ + Implemented to make sorted available upon CharsetMatches items. + """ + if not isinstance(other, CharsetMatch): + raise ValueError + + chaos_difference: float = abs(self.chaos - other.chaos) + coherence_difference: float = abs(self.coherence - other.coherence) + + # Below 1% difference --> Use Coherence + if chaos_difference < 0.01 and coherence_difference > 0.02: + return self.coherence > other.coherence + elif chaos_difference < 0.01 and coherence_difference <= 0.02: + # When having a difficult decision, use the result that decoded as many multi-byte as possible. + # preserve RAM usage! + if len(self._payload) >= TOO_BIG_SEQUENCE: + return self.chaos < other.chaos + return self.multi_byte_usage > other.multi_byte_usage + + return self.chaos < other.chaos + + @property + def multi_byte_usage(self) -> float: + return 1.0 - (len(str(self)) / len(self.raw)) + + def __str__(self) -> str: + # Lazy Str Loading + if self._string is None: + self._string = str(self._payload, self._encoding, "strict") + return self._string + + def __repr__(self) -> str: + return f"" + + def add_submatch(self, other: CharsetMatch) -> None: + if not isinstance(other, CharsetMatch) or other == self: + raise ValueError( + "Unable to add instance <{}> as a submatch of a CharsetMatch".format( + other.__class__ + ) + ) + + other._string = None # Unload RAM usage; dirty trick. + self._leaves.append(other) + + @property + def encoding(self) -> str: + return self._encoding + + @property + def encoding_aliases(self) -> list[str]: + """ + Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. + """ + also_known_as: list[str] = [] + for u, p in aliases.items(): + if self.encoding == u: + also_known_as.append(p) + elif self.encoding == p: + also_known_as.append(u) + return also_known_as + + @property + def bom(self) -> bool: + return self._has_sig_or_bom + + @property + def byte_order_mark(self) -> bool: + return self._has_sig_or_bom + + @property + def languages(self) -> list[str]: + """ + Return the complete list of possible languages found in decoded sequence. + Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. + """ + return [e[0] for e in self._languages] + + @property + def language(self) -> str: + """ + Most probable language found in decoded sequence. If none were detected or inferred, the property will return + "Unknown". + """ + if not self._languages: + # Trying to infer the language based on the given encoding + # Its either English or we should not pronounce ourselves in certain cases. + if "ascii" in self.could_be_from_charset: + return "English" + + # doing it there to avoid circular import + from charset_normalizer.cd import encoding_languages, mb_encoding_languages + + languages = ( + mb_encoding_languages(self.encoding) + if is_multi_byte_encoding(self.encoding) + else encoding_languages(self.encoding) + ) + + if len(languages) == 0 or "Latin Based" in languages: + return "Unknown" + + return languages[0] + + return self._languages[0][0] + + @property + def chaos(self) -> float: + return self._mean_mess_ratio + + @property + def coherence(self) -> float: + if not self._languages: + return 0.0 + return self._languages[0][1] + + @property + def percent_chaos(self) -> float: + return round(self.chaos * 100, ndigits=3) + + @property + def percent_coherence(self) -> float: + return round(self.coherence * 100, ndigits=3) + + @property + def raw(self) -> bytes: + """ + Original untouched bytes. + """ + return self._payload + + @property + def submatch(self) -> list[CharsetMatch]: + return self._leaves + + @property + def has_submatch(self) -> bool: + return len(self._leaves) > 0 + + @property + def alphabets(self) -> list[str]: + if self._unicode_ranges is not None: + return self._unicode_ranges + # list detected ranges + detected_ranges: list[str | None] = [unicode_range(char) for char in str(self)] + # filter and sort + self._unicode_ranges = sorted(list({r for r in detected_ranges if r})) + return self._unicode_ranges + + @property + def could_be_from_charset(self) -> list[str]: + """ + The complete list of encoding that output the exact SAME str result and therefore could be the originating + encoding. + This list does include the encoding available in property 'encoding'. + """ + return [self._encoding] + [m.encoding for m in self._leaves] + + def output(self, encoding: str = "utf_8") -> bytes: + """ + Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. + Any errors will be simply ignored by the encoder NOT replaced. + """ + if self._output_encoding is None or self._output_encoding != encoding: + self._output_encoding = encoding + decoded_string = str(self) + if ( + self._preemptive_declaration is not None + and self._preemptive_declaration.lower() + not in ["utf-8", "utf8", "utf_8"] + ): + patched_header = sub( + RE_POSSIBLE_ENCODING_INDICATION, + lambda m: m.string[m.span()[0] : m.span()[1]].replace( + m.groups()[0], iana_name(self._output_encoding) # type: ignore[arg-type] + ), + decoded_string[:8192], + 1, + ) + + decoded_string = patched_header + decoded_string[8192:] + + self._output_payload = decoded_string.encode(encoding, "replace") + + return self._output_payload # type: ignore + + @property + def fingerprint(self) -> str: + """ + Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. + """ + return sha256(self.output()).hexdigest() + + +class CharsetMatches: + """ + Container with every CharsetMatch items ordered by default from most probable to the less one. + Act like a list(iterable) but does not implements all related methods. + """ + + def __init__(self, results: list[CharsetMatch] | None = None): + self._results: list[CharsetMatch] = sorted(results) if results else [] + + def __iter__(self) -> Iterator[CharsetMatch]: + yield from self._results + + def __getitem__(self, item: int | str) -> CharsetMatch: + """ + Retrieve a single item either by its position or encoding name (alias may be used here). + Raise KeyError upon invalid index or encoding not present in results. + """ + if isinstance(item, int): + return self._results[item] + if isinstance(item, str): + item = iana_name(item, False) + for result in self._results: + if item in result.could_be_from_charset: + return result + raise KeyError + + def __len__(self) -> int: + return len(self._results) + + def __bool__(self) -> bool: + return len(self._results) > 0 + + def append(self, item: CharsetMatch) -> None: + """ + Insert a single match. Will be inserted accordingly to preserve sort. + Can be inserted as a submatch. + """ + if not isinstance(item, CharsetMatch): + raise ValueError( + "Cannot append instance '{}' to CharsetMatches".format( + str(item.__class__) + ) + ) + # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage) + if len(item.raw) < TOO_BIG_SEQUENCE: + for match in self._results: + if match.fingerprint == item.fingerprint and match.chaos == item.chaos: + match.add_submatch(item) + return + self._results.append(item) + self._results = sorted(self._results) + + def best(self) -> CharsetMatch | None: + """ + Simply return the first match. Strict equivalent to matches[0]. + """ + if not self._results: + return None + return self._results[0] + + def first(self) -> CharsetMatch | None: + """ + Redundant method, call the method best(). Kept for BC reasons. + """ + return self.best() + + +CoherenceMatch = Tuple[str, float] +CoherenceMatches = List[CoherenceMatch] + + +class CliDetectionResult: + def __init__( + self, + path: str, + encoding: str | None, + encoding_aliases: list[str], + alternative_encodings: list[str], + language: str, + alphabets: list[str], + has_sig_or_bom: bool, + chaos: float, + coherence: float, + unicode_path: str | None, + is_preferred: bool, + ): + self.path: str = path + self.unicode_path: str | None = unicode_path + self.encoding: str | None = encoding + self.encoding_aliases: list[str] = encoding_aliases + self.alternative_encodings: list[str] = alternative_encodings + self.language: str = language + self.alphabets: list[str] = alphabets + self.has_sig_or_bom: bool = has_sig_or_bom + self.chaos: float = chaos + self.coherence: float = coherence + self.is_preferred: bool = is_preferred + + @property + def __dict__(self) -> dict[str, Any]: # type: ignore + return { + "path": self.path, + "encoding": self.encoding, + "encoding_aliases": self.encoding_aliases, + "alternative_encodings": self.alternative_encodings, + "language": self.language, + "alphabets": self.alphabets, + "has_sig_or_bom": self.has_sig_or_bom, + "chaos": self.chaos, + "coherence": self.coherence, + "unicode_path": self.unicode_path, + "is_preferred": self.is_preferred, + } + + def to_json(self) -> str: + return dumps(self.__dict__, ensure_ascii=True, indent=4) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/py.typed b/tools/retro-library/charset_normalizer/charset_normalizer/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/utils.py b/tools/retro-library/charset_normalizer/charset_normalizer/utils.py new file mode 100644 index 000000000..4498d3c6c --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/utils.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +import importlib +import logging +import unicodedata +from codecs import IncrementalDecoder +from encodings.aliases import aliases +from functools import lru_cache +from re import findall +from typing import Generator + +from _multibytecodec import ( # type: ignore[import-not-found,import] + MultibyteIncrementalDecoder, +) + +from .constant import ( + ENCODING_MARKS, + IANA_SUPPORTED_SIMILAR, + RE_POSSIBLE_ENCODING_INDICATION, + UNICODE_RANGES_COMBINED, + UNICODE_SECONDARY_RANGE_KEYWORD, + UTF8_MAXIMAL_ALLOCATION, +) + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_accentuated(character: str) -> bool: + try: + description: str = unicodedata.name(character) + except ValueError: + return False + return ( + "WITH GRAVE" in description + or "WITH ACUTE" in description + or "WITH CEDILLA" in description + or "WITH DIAERESIS" in description + or "WITH CIRCUMFLEX" in description + or "WITH TILDE" in description + or "WITH MACRON" in description + or "WITH RING ABOVE" in description + ) + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def remove_accent(character: str) -> str: + decomposed: str = unicodedata.decomposition(character) + if not decomposed: + return character + + codes: list[str] = decomposed.split(" ") + + return chr(int(codes[0], 16)) + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def unicode_range(character: str) -> str | None: + """ + Retrieve the Unicode range official name from a single character. + """ + character_ord: int = ord(character) + + for range_name, ord_range in UNICODE_RANGES_COMBINED.items(): + if character_ord in ord_range: + return range_name + + return None + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_latin(character: str) -> bool: + try: + description: str = unicodedata.name(character) + except ValueError: + return False + return "LATIN" in description + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_punctuation(character: str) -> bool: + character_category: str = unicodedata.category(character) + + if "P" in character_category: + return True + + character_range: str | None = unicode_range(character) + + if character_range is None: + return False + + return "Punctuation" in character_range + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_symbol(character: str) -> bool: + character_category: str = unicodedata.category(character) + + if "S" in character_category or "N" in character_category: + return True + + character_range: str | None = unicode_range(character) + + if character_range is None: + return False + + return "Forms" in character_range and character_category != "Lo" + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_emoticon(character: str) -> bool: + character_range: str | None = unicode_range(character) + + if character_range is None: + return False + + return "Emoticons" in character_range or "Pictographs" in character_range + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_separator(character: str) -> bool: + if character.isspace() or character in {"｜", "+", "<", ">"}: + return True + + character_category: str = unicodedata.category(character) + + return "Z" in character_category or character_category in {"Po", "Pd", "Pc"} + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_case_variable(character: str) -> bool: + return character.islower() != character.isupper() + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_cjk(character: str) -> bool: + try: + character_name = unicodedata.name(character) + except ValueError: + return False + + return "CJK" in character_name + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_hiragana(character: str) -> bool: + try: + character_name = unicodedata.name(character) + except ValueError: + return False + + return "HIRAGANA" in character_name + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_katakana(character: str) -> bool: + try: + character_name = unicodedata.name(character) + except ValueError: + return False + + return "KATAKANA" in character_name + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_hangul(character: str) -> bool: + try: + character_name = unicodedata.name(character) + except ValueError: + return False + + return "HANGUL" in character_name + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_thai(character: str) -> bool: + try: + character_name = unicodedata.name(character) + except ValueError: + return False + + return "THAI" in character_name + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_arabic(character: str) -> bool: + try: + character_name = unicodedata.name(character) + except ValueError: + return False + + return "ARABIC" in character_name + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_arabic_isolated_form(character: str) -> bool: + try: + character_name = unicodedata.name(character) + except ValueError: + return False + + return "ARABIC" in character_name and "ISOLATED FORM" in character_name + + +@lru_cache(maxsize=len(UNICODE_RANGES_COMBINED)) +def is_unicode_range_secondary(range_name: str) -> bool: + return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD) + + +@lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) +def is_unprintable(character: str) -> bool: + return ( + character.isspace() is False # includes \n \t \r \v + and character.isprintable() is False + and character != "\x1A" # Why? Its the ASCII substitute character. + and character != "\ufeff" # bug discovered in Python, + # Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space. + ) + + +def any_specified_encoding(sequence: bytes, search_zone: int = 8192) -> str | None: + """ + Extract using ASCII-only decoder any specified encoding in the first n-bytes. + """ + if not isinstance(sequence, bytes): + raise TypeError + + seq_len: int = len(sequence) + + results: list[str] = findall( + RE_POSSIBLE_ENCODING_INDICATION, + sequence[: min(seq_len, search_zone)].decode("ascii", errors="ignore"), + ) + + if len(results) == 0: + return None + + for specified_encoding in results: + specified_encoding = specified_encoding.lower().replace("-", "_") + + encoding_alias: str + encoding_iana: str + + for encoding_alias, encoding_iana in aliases.items(): + if encoding_alias == specified_encoding: + return encoding_iana + if encoding_iana == specified_encoding: + return encoding_iana + + return None + + +@lru_cache(maxsize=128) +def is_multi_byte_encoding(name: str) -> bool: + """ + Verify is a specific encoding is a multi byte one based on it IANA name + """ + return name in { + "utf_8", + "utf_8_sig", + "utf_16", + "utf_16_be", + "utf_16_le", + "utf_32", + "utf_32_le", + "utf_32_be", + "utf_7", + } or issubclass( + importlib.import_module(f"encodings.{name}").IncrementalDecoder, + MultibyteIncrementalDecoder, + ) + + +def identify_sig_or_bom(sequence: bytes) -> tuple[str | None, bytes]: + """ + Identify and extract SIG/BOM in given sequence. + """ + + for iana_encoding in ENCODING_MARKS: + marks: bytes | list[bytes] = ENCODING_MARKS[iana_encoding] + + if isinstance(marks, bytes): + marks = [marks] + + for mark in marks: + if sequence.startswith(mark): + return iana_encoding, mark + + return None, b"" + + +def should_strip_sig_or_bom(iana_encoding: str) -> bool: + return iana_encoding not in {"utf_16", "utf_32"} + + +def iana_name(cp_name: str, strict: bool = True) -> str: + cp_name = cp_name.lower().replace("-", "_") + + encoding_alias: str + encoding_iana: str + + for encoding_alias, encoding_iana in aliases.items(): + if cp_name in [encoding_alias, encoding_iana]: + return encoding_iana + + if strict: + raise ValueError(f"Unable to retrieve IANA for '{cp_name}'") + + return cp_name + + +def range_scan(decoded_sequence: str) -> list[str]: + ranges: set[str] = set() + + for character in decoded_sequence: + character_range: str | None = unicode_range(character) + + if character_range is None: + continue + + ranges.add(character_range) + + return list(ranges) + + +def cp_similarity(iana_name_a: str, iana_name_b: str) -> float: + if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b): + return 0.0 + + decoder_a = importlib.import_module(f"encodings.{iana_name_a}").IncrementalDecoder + decoder_b = importlib.import_module(f"encodings.{iana_name_b}").IncrementalDecoder + + id_a: IncrementalDecoder = decoder_a(errors="ignore") + id_b: IncrementalDecoder = decoder_b(errors="ignore") + + character_match_count: int = 0 + + for i in range(255): + to_be_decoded: bytes = bytes([i]) + if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded): + character_match_count += 1 + + return character_match_count / 254 + + +def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool: + """ + Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using + the function cp_similarity. + """ + return ( + iana_name_a in IANA_SUPPORTED_SIMILAR + and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a] + ) + + +def set_logging_handler( + name: str = "charset_normalizer", + level: int = logging.INFO, + format_string: str = "%(asctime)s | %(levelname)s | %(message)s", +) -> None: + logger = logging.getLogger(name) + logger.setLevel(level) + + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(format_string)) + logger.addHandler(handler) + + +def cut_sequence_chunks( + sequences: bytes, + encoding_iana: str, + offsets: range, + chunk_size: int, + bom_or_sig_available: bool, + strip_sig_or_bom: bool, + sig_payload: bytes, + is_multi_byte_decoder: bool, + decoded_payload: str | None = None, +) -> Generator[str, None, None]: + if decoded_payload and is_multi_byte_decoder is False: + for i in offsets: + chunk = decoded_payload[i : i + chunk_size] + if not chunk: + break + yield chunk + else: + for i in offsets: + chunk_end = i + chunk_size + if chunk_end > len(sequences) + 8: + continue + + cut_sequence = sequences[i : i + chunk_size] + + if bom_or_sig_available and strip_sig_or_bom is False: + cut_sequence = sig_payload + cut_sequence + + chunk = cut_sequence.decode( + encoding_iana, + errors="ignore" if is_multi_byte_decoder else "strict", + ) + + # multi-byte bad cutting detector and adjustment + # not the cleanest way to perform that fix but clever enough for now. + if is_multi_byte_decoder and i > 0: + chunk_partial_size_chk: int = min(chunk_size, 16) + + if ( + decoded_payload + and chunk[:chunk_partial_size_chk] not in decoded_payload + ): + for j in range(i, i - 4, -1): + cut_sequence = sequences[j:chunk_end] + + if bom_or_sig_available and strip_sig_or_bom is False: + cut_sequence = sig_payload + cut_sequence + + chunk = cut_sequence.decode(encoding_iana, errors="ignore") + + if chunk[:chunk_partial_size_chk] in decoded_payload: + break + + yield chunk diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/version.py b/tools/retro-library/charset_normalizer/charset_normalizer/version.py new file mode 100644 index 000000000..f85e8929e --- /dev/null +++ b/tools/retro-library/charset_normalizer/charset_normalizer/version.py @@ -0,0 +1,8 @@ +""" +Expose version +""" + +from __future__ import annotations + +__version__ = "3.4.1" +VERSION = __version__.split(".") diff --git a/tools/retro-library/charset_normalizer/data/NOTICE.md b/tools/retro-library/charset_normalizer/data/NOTICE.md new file mode 100644 index 000000000..4154617ca --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/NOTICE.md @@ -0,0 +1,9 @@ +Included and Redistributed Files +--------------------------------- + +17 files are included in the source distribution tar. They are used to verify the standard functions of +this library. They are mandatory to run `pytest` but not required to make the lib usable after install. +They DO NOT guarantee that the detection-coverage will not regress. + +Those are EITHER pulled from Wikipedia _(CC-BY-SA)_ OR public domain archive. +You SHALL NOT modify any of those files without explicit approval. diff --git a/tools/retro-library/charset_normalizer/data/sample-arabic-1.txt b/tools/retro-library/charset_normalizer/data/sample-arabic-1.txt new file mode 100644 index 000000000..5648760e2 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-arabic-1.txt @@ -0,0 +1,6 @@ + " " " " : ( ) ( ) ( ). + + : " " " " " " ߡ . + + ( : ) ( : ). ( : Morroch) + diff --git a/tools/retro-library/charset_normalizer/data/sample-arabic.txt b/tools/retro-library/charset_normalizer/data/sample-arabic.txt new file mode 100644 index 000000000..4c18892b8 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-arabic.txt @@ -0,0 +1,6 @@ +بالموازاة مع ذلك وللإشارة إلى المنطقة المغاربية بشكل عام، كان المؤرخون العرب في القرون الوسطى يستعملون لفظ "بلاد المغرب" بينما الأوروبيون يستعملون لفظ "الساحل البربري" للدلالة على ثلاثة أقاليم: المغرب الأدنى (إفريقية أو تونس الحالية)، المغرب الأوسط (الجزائر الحالية)، المغرب الأقصى (المملكة المغربية الحالية). + +أحيانًا كان يُشار للبلاد بتسمية مرتبطة بعاصمتها: كـ "موريطنية الطنجية" التي كانت عاصمتها طنجة وكذا "مملكة مراكش" و"مملكة فاس" نسبة إلى عواصمها المعروفة آنذاك، وكانت الظهائر والمعاهدات الدولية يوقّعها سلاطين المغرب تارة باسم سلطان مراكش وتارة باسم سلطان فاس. + +تمت الإشارة للبلاد لاحقًا باسم المغرب الأقصى باللغة العربية حيث اعتَقد الناس في العالم القديم أن الشمس تشرق من اليابان (باللغة الصينية نيهون: مكان شروق الشمس) وتغرب في المملكة المغربية (باللغة العربية المغرب: مكان غروب الشمس). بينما اشتَقت البلاد اسمها في اللغات الأوروبية من الكلمة اللاتينية مرك (باللغة اللاتينية: Morroch) وهي تصحيف + diff --git a/tools/retro-library/charset_normalizer/data/sample-bulgarian.txt b/tools/retro-library/charset_normalizer/data/sample-bulgarian.txt new file mode 100755 index 000000000..7a3ad68a3 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-bulgarian.txt @@ -0,0 +1,7 @@ +Член 26 +1. Bсеки човек има право на образование. Oбразованието трябва да бъде безплатно, поне що се отнася до началното и основното образование. Hачалното образование трябва да бъде задължително. Tехническото и професионалното образование трябва да бъдат общодостъпни, а висшето образование трябва да бъде еднакво достъпно за всички на основата на техните способности. +2. Oбразованието трябва да бъде насочено към цялостно развитие на човешката личност и заcилване на уважението към правата на човека и основните свободи. Tо трябва да съдейства за разбирателството, тъпримостта и приятелството между всички народи, расови или религиозни групи, както и за осъществяване дейността на Oрганизацията на Oбединените нации за поддържане на мира. +3. Pодителите имат право, с приоритет, да избират вида образование, което да получат техните деца. +Член 27 +1. Bсеки човек има право свободно да участва в културния живот на обществото, да се наслаждава на изкуствата, да участва в научния напредък и да се ползва от неговите достижения. +2. Bсеки човек има право на закрила на моралните и материалните си интереси, които са резултат от каквото и да е научно, литературно или художествено произведение, на което той е автор. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/data/sample-chinese.txt b/tools/retro-library/charset_normalizer/data/sample-chinese.txt new file mode 100644 index 000000000..760df5af8 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-chinese.txt @@ -0,0 +1,14 @@ +j]Wikipedia^̡A¦FѤUB|BHӡAѦʬjCl@̡AC|]C + +FG~QGܤ@AΤv~GQEAUyGʤQAXOCʸUءFGQjKA^LGʸUCxDѤUӦ@ӦFNUAXBHġ@Aj_jC + +@AXjոܺCmlnJuAô]vAmnJuAl]vAHXANôl]C´AHڥA]C + +joAPDݵoFpsDBХBBӾǡByBwCئhiw\שsF\׽֡AѱoPġA_ɨaHAMhҸչAKGC + +Z򤧵AҾڪ̡AꭲۥѤɳ\iijAGiۥѼsǤѤUաC + +娥l~CiAoGd@ʤKQEC + +commons: + nHMTA@ɡJjC diff --git a/tools/retro-library/charset_normalizer/data/sample-english.bom.txt b/tools/retro-library/charset_normalizer/data/sample-english.bom.txt new file mode 100755 index 000000000..7c87e9489 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-english.bom.txt @@ -0,0 +1,35 @@ +1 +00:00:06,500 --> 00:00:09,000 +About 2 months ago I found myself on +the comment section of YouTube + +2 +00:00:11,000 --> 00:00:17,000 +And I was commenting, +unfortunately I was commenting, +on a video about the famous Ayn Rand + +3 +00:00:19,000 --> 00:00:24,000 +And I +posted underneath against +this woman's tirades, +against what is essentially +the human race. + +4 +00:00:25,000 --> 00:00:31,000 +that, this monetary system seems to have no point, seems to actually hinder people + +5 +00:00:31,000 --> 00:00:36,000 +and hinder progress, and one of the responses I got, I didn't answer it, was: + +6 +00:00:37,000 --> 00:00:43,000 +what actually money creates is an incentive to invent the new items, that's the driving force behind it + +7 +00:00:43,000 --> 00:00:50,000 +So what I thought I do is instead if answering on a YouTube comment is organize a global awareness day + diff --git a/tools/retro-library/charset_normalizer/data/sample-french-1.txt b/tools/retro-library/charset_normalizer/data/sample-french-1.txt new file mode 100644 index 000000000..f87bdf715 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-french-1.txt @@ -0,0 +1,59 @@ +JEAN-BAPTISTE POQUELIN MOLIRE +N LE 15 JANVIER 1622, MORT LE 17 FVRIER 1673 + + +Quel est le plus grand des crivains de mon rgne? demandait Louis XIV + Boileau.--Sire, c'est Molire. + +Non-seulement Despraux ne se trompait pas, mais de tous les crivains +que la France a produits, sans excepter Voltaire lui-mme, imprgn de +l'esprit anglais par son sjour Londres, c'est incontestablement +Molire ou Poquelin qui reproduit avec l'exactitude la plus vive et la +plus complte le fond du gnie franais. + +En raison de cette identit de son gnie avec le ntre, il exera sur +l'poque subsquente, sur le dix-huitime sicle, sur l'poque mme o +nous crivons, la plus active, la plus redoutable influence. Tout ce +qu'il a voulu dtruire est en ruine. Les types qu'il a crs ne peuvent +mourir. Le sens de la vie pratique, qu'il a recommand d'aprs Gassendi, +a fini par l'emporter sur les ides qui imposaient la socit +franaise. Il n'y a pas de superstition qu'il n'ait attaque, pas de +crdulit qu'il n'ait saisie corps corps pour la terrasser, pas de +formule qu'il ne se soit efforc de dtruire. A-t-il, comme l'exprime si +bien Swift, _dchir l'toffe avec la doublure_? l'histoire le dira. Ce +qui est certain, c'est que l'lve de Lucrce, le protg de Louis XIV, +poursuivait un but dtermin vers lequel il a march d'un pas ferme, +obstin, tantt foulant aux pieds les obstacles, tantt les tournant +avec adresse. Le sujet de _Tartuffe_ est dans Lucrce; Lucrce +appartient ce vers, vritable devise de Molire: + + _Et religionis..... nodos solvere curo[1]._ + +La puissance de Molire sur les esprits a t telle, qu'une lgende +inexacte, calomnieuse de son vivant, romanesque aprs sa mort, s'est +forme autour de cette gloire populaire. Il est un mythe comme Jules +Csar et Apollon. + + [1] Ce que je veux, c'est rompre les entraves qui nous enchanent + (_religionis.... quod religat_). + +Dates, vnements, ralits, souvenirs, sont venus se confondre dans un +inextricable chaos o la figure de Molire a disparu. Tous les vices +jusqu' l'ivrognerie, jusqu' l'inceste et au vol, lui furent imputs de +son vivant. Les vertus les plus thres lui furent attribues par les +prtres de son culte. Homme d'action, sans cesse en face du public, du +roi ou de sa troupe, occup de son gouvernement et de la cration de ses +uvres, il n'a laiss aucune trace de sa propre vie, aucun document +biographique, peine une lettre. Les pamphlets pour et contre lui +composaient dj une bibliothque, lorsqu'un couteur aux portes, nomm +Grimarest, collecteur d'anas, aimant l'exagration des rcits et +incapable de critique, prtendit, trente-deux ans aprs la mort du +comdien populaire, raconter et expliquer sa vie. Vers la mme poque, +une comdienne, ce que l'on croit du moins, force de se rfugier en +Hollande, jetait dans un libelle les souvenirs de coulisse qu'elle avait +pu recueillir sur l'intrieur du mnage de Molire et de sa femme. Enfin +quelques dtails authentiques, sems dans l'dition de ses uvres +publie par Lagrange en 1682, compltent l'ensemble des documents +comtemporains qui ont servi de base cette lgende de Molire, +excellente consulter, mais qu'il est bon de soumettre l'examen le +plus scrupuleux. diff --git a/tools/retro-library/charset_normalizer/data/sample-french.txt b/tools/retro-library/charset_normalizer/data/sample-french.txt new file mode 100644 index 000000000..640431000 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-french.txt @@ -0,0 +1,59 @@ +JEAN-BAPTISTE POQUELIN MOLIÈRE +NÉ LE 15 JANVIER 1622, MORT LE 17 FÉVRIER 1673 + + +«Quel est le plus grand des écrivains de mon règne? demandait Louis XIV +à Boileau.--Sire, c'est Molière.» + +Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains +que la France a produits, sans excepter Voltaire lui-même, imprégné de +l'esprit anglais par son séjour à Londres, c'est incontestablement +Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la +plus complète le fond du génie français. + +En raison de cette identité de son génie avec le nôtre, il exerça sur +l'époque subséquente, sur le dix-huitième siècle, sur l'époque même où +nous écrivons, la plus active, la plus redoutable influence. Tout ce +qu'il a voulu détruire est en ruine. Les types qu'il a créés ne peuvent +mourir. Le sens de la vie pratique, qu'il a recommandé d'après Gassendi, +a fini par l'emporter sur les idées qui imposaient à la société +française. Il n'y a pas de superstition qu'il n'ait attaquée, pas de +crédulité qu'il n'ait saisie corps à corps pour la terrasser, pas de +formule qu'il ne se soit efforcé de détruire. A-t-il, comme l'exprime si +bien Swift, _déchiré l'étoffe avec la doublure_? l'histoire le dira. Ce +qui est certain, c'est que l'élève de Lucrèce, le protégé de Louis XIV, +poursuivait un but déterminé vers lequel il a marché d'un pas ferme, +obstiné, tantôt foulant aux pieds les obstacles, tantôt les tournant +avec adresse. Le sujet de _Tartuffe_ est dans Lucrèce; à Lucrèce +appartient ce vers, véritable devise de Molière: + + _Et religionis..... nodos solvere curo[1]._ + +La puissance de Molière sur les esprits a été telle, qu'une légende +inexacte, calomnieuse de son vivant, romanesque après sa mort, s'est +formée autour de cette gloire populaire. Il est un mythe comme Jules +César et Apollon. + + [1] Ce que je veux, c'est rompre les entraves qui nous enchaînent + (_religionis.... quod religat_). + +Dates, événements, réalités, souvenirs, sont venus se confondre dans un +inextricable chaos où la figure de Molière a disparu. Tous les vices +jusqu'à l'ivrognerie, jusqu'à l'inceste et au vol, lui furent imputés de +son vivant. Les vertus les plus éthérées lui furent attribuées par les +prêtres de son culte. Homme d'action, sans cesse en face du public, du +roi ou de sa troupe, occupé de son gouvernement et de la création de ses +œuvres, il n'a laissé aucune trace de sa propre vie, aucun document +biographique, à peine une lettre. Les pamphlets pour et contre lui +composaient déjà une bibliothèque, lorsqu'un écouteur aux portes, nommé +Grimarest, collecteur d'anas, aimant l'exagération des récits et +incapable de critique, prétendit, trente-deux ans après la mort du +comédien populaire, raconter et expliquer sa vie. Vers la même époque, +une comédienne, à ce que l'on croit du moins, forcée de se réfugier en +Hollande, jetait dans un libelle les souvenirs de coulisse qu'elle avait +pu recueillir sur l'intérieur du ménage de Molière et de sa femme. Enfin +quelques détails authentiques, semés dans l'édition de ses œuvres +publiée par Lagrange en 1682, complètent l'ensemble des documents +comtemporains qui ont servi de base à cette légende de Molière, +excellente à consulter, mais qu'il est bon de soumettre à l'examen le +plus scrupuleux. diff --git a/tools/retro-library/charset_normalizer/data/sample-greek-2.txt b/tools/retro-library/charset_normalizer/data/sample-greek-2.txt new file mode 100644 index 000000000..bfddcfaca --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-greek-2.txt @@ -0,0 +1 @@ + 12 , . , , . - , , , , , . 20 . diff --git a/tools/retro-library/charset_normalizer/data/sample-greek.txt b/tools/retro-library/charset_normalizer/data/sample-greek.txt new file mode 100644 index 000000000..bfddcfaca --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-greek.txt @@ -0,0 +1 @@ + 12 , . , , . - , , , , , . 20 . diff --git a/tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt b/tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt new file mode 100644 index 000000000..ea804346f --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt @@ -0,0 +1 @@ + . , . (), , . (), (). diff --git a/tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt b/tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt new file mode 100755 index 000000000..ea804346f --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt @@ -0,0 +1 @@ + . , . (), , . (), (). diff --git a/tools/retro-library/charset_normalizer/data/sample-korean.txt b/tools/retro-library/charset_normalizer/data/sample-korean.txt new file mode 100644 index 000000000..6f632b190 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-korean.txt @@ -0,0 +1 @@ + ڵ 絵 ż ƴ, ΰμ ߱Ѵ. 20 Ŀ , 縯 ȯ ̴ ũλ ̳  ǰ ִ. ѹα ѱ ⵶ ( ؿ) ũλ Ͽ, Ұϰ ִ. diff --git a/tools/retro-library/charset_normalizer/data/sample-polish.txt b/tools/retro-library/charset_normalizer/data/sample-polish.txt new file mode 100644 index 000000000..9e506c26d --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-polish.txt @@ -0,0 +1,204 @@ +"source";"target" +"REF.-2";"POLISH" +"KW-P00-01";"SYSTEM VIDEODOMOFONOWY MEET" +"KW-P00-02";"URZĄDZENIE" +"KW-P00-03";"OGÓLNE" +"KW-P00-04";"SIEĆ" +"KW-P00-05";"KD" +"KW-P00-06";"ROZP. TWARZY." +"KW-P00-07";"KAMERY IP" +"KW-P00-08";"SIP" +"KW-P00-09";"SIP TRUNK" +"KW-P00-10";"PRZEKIEROWANIA" +"KW-P00-11";"ZAAWANSOWANE" +"KW-P00-12";"KOD PIN" +"KW-P00-13";"WECHAT QR" +"KW-P00-14";"PRZYWRACAĆ" +"KW-P00-16";"WINDA" +"KW-P01-01";"INFORMACJE O URZĄDZENIU" +"KW-P01-02";"PANEL VIDEO FOOBAR KIN" +"KW-P01-03";"FIRMWARE: V02.10" +"KW-P01-04";"URZĄDZENIE: PANEL BLOKOWY-CYFROWY 001-02" +"KW-P01-05";"URZĄDZENIE: PANEL BLOKOWY PRZYCISKI 020-02" +"KW-P01-06";"URZĄDZENIE: PANEL GŁÓWNY 01" +"KW-P01-07";"URZĄDZENIE: PANEL 1W 006-0102-01" +"KW-P01-08";"NUMER SERYJNY:" +"KW-P01-09";"MAC:" +"KW-P01-10";"IP:" +"KW-P01-11";"COPYRIGHT © FOOBAR " +"KW-P01-12";"www.example.com" +"KW-P02-01";"USTAWIENIA GŁÓWNE" +"KW-P02-02";"TYP:" +"KW-P02-03";"PANEL GŁÓWNY" +"KW-P02-04";"CYFROWY P. BLOKOWY" +"KW-P02-05";"P. BLOK. PRZYCISKI" +"KW-P02-06";"PANEL 1NR" +"KW-P02-07";"BLOK:" +"KW-P02-08";"LOKAL:" +"KW-P02-09";"MONIT WYŚWIETLACZA:" +"KW-P02-10";"THIS INTERFACE IS NOT ENABLED" +"KW-P02-11";"NUMER PANELU:" +"KW-P02-12";"NAZWA URZĄDZENIA:" +"KW-P02-13";"(≤16 ZNAKÓW)" +"KW-P02-14";"JĘZYK:" +"KW-P02-15";"ENGLISH" +"KW-P02-16";"中文" +"KW-P02-17";"ESPAÑOL" +"KW-P02-18";"РУССКИЙ" +"KW-P02-19";"DEUTSCH" +"KW-P02-20";"TÜRKÇE" +"KW-P02-21";"POLSKI" +"KW-P02-22";"עברית" +"KW-P02-23";"FRANÇAIS" +"KW-P02-24";"فارسی" +"KW-P02-25";"GŁOŚNOŚĆ PANELU:" +"KW-P02-26";"JASNOŚĆ" +"KW-P02-27";"ROZDZIELCZOŚĆ VIDEO:" +"KW-P02-28";"TRYB PRZEKIEROWANIA SIP:" +"KW-P02-29";"SEKWENCYJNE" +"KW-P02-30";"JEDNOCZESNE" +"KW-P02-31";"PORTIER:" +"KW-P02-32";"PORTIERNIA 1:" +"KW-P02-33";"PORTIERNIA 2:" +"KW-P02-34";"USTAW. DATY I CZASU" +"KW-P02-35";"FORMAT DATY:" +"KW-P02-36";"DATA:" +"KW-P02-37";"CZAS:" +"KW-P02-38";"STREFA CZASOWA:" +"KW-P02-39";"ZAPISZ" +"KW-P02-40";"BŁĘDNE DANE" +"KW-P02-41";"KLAWIATURA ALFANUM.:" +"KW-P02-42";"KOMUNIKAT OTWARCIA DRZWI:" +"KW-P02-43";"WYGASZACZ EKRANU:" +"KW-P02-44";"WSPARCIE:" +"KW-P02-45";"OCZEKIWANIE" +"KW-P02-46";"POŁĄCZENIE" +"KW-P02-47";"WSPARCIE" +"KW-P02-48";"lista" +"KW-P02-49";"DST:" +"KW-P02-57";"TŁO:" +"KW-P02-58";"CIEMNE" +"KW-P02-59";"JASNE" +"KW-P02-60";"IMPORT" +"KW-P02-61";"EKSPORT" +"KW-P02-62";"USUŃ" +"KW-P02-63";"WYBIERZ PRAWIDŁOWY PLIK PNG" +"KW-P02-64";"IMPORTUJ" +"KW-P02-65";"WYSYŁANIE ZAKOŃCZONE" +"KW-P02-66";"BRAK OBRAZU" +"KW-P02-67";"USUNIĘTE" +"KW-P02-68";"BŁĄD USUWANIA" +"KW-P03-01";"USTAWIENIA SIECI" +"KW-P03-02";"IP:" +"KW-P03-03";"MASKA:" +"KW-P03-04";"BRAMA:" +"KW-P03-05";"DNS:" +"KW-P03-06";"SOFTWARE IP:" +"KW-P03-07";"SW. PIN:" +"KW-P03-08";"ZAPISZ" +"KW-P04-01";"USTAWIENIA KONTROLI DOSTĘPU" +"KW-P04-02";"PRZYCISK EGRESS:" +"KW-P04-03";"CZAS ELEKTROZACZEPU:" +"KW-P04-04";"CZAS KONTAKTRONU:" +"KW-P04-05";"REF.1491 4 RELAY:" +"KW-P04-06";"CZAS ELEKTROZACZEPU:" +"KW-P04-07";"CZAS KONTAKTRONU:" +"KW-P04-08";"KARTA ADMINISTRATORA:" +"KW-P04-09";"ROZBRAJANIE KARTĄ:" +"KW-P04-10";"MONITY KART:" +"KW-P04-11";"KOD GOŚCIA:" +"KW-P04-12";"KOD DOSTĘPU:" +"KW-P04-13";"#1" +"KW-P04-14";"#2" +"KW-P04-15";"#3" +"KW-P04-16";"#4" +"KW-P04-17";"ALARM DRZWI" +"KW-P04-18";"GWAŁTOWNY ALARM OTWARCIA" +"KW-P04-19";"WIEGAND:" +"KW-P04-20";"BURST" +"KW-P04-21";"26-BIT" +"KW-P04-22";"FACILITY:" +"KW-P04-24";"ZAPISZ" +"KW-P04-25";"WYŁĄCZONY" +"KW-P04-26";"REF.1490 2 RELAY:" +"KW-P04-27";"KOD QR:" +"KW-P04-28";"WIEGAND:" +"KW-P04-29";"26-BIT" +"KW-P04-30";"34-BIT" +"KW-P04-31";"KOD MIEJSCA:" +"KW-P04-32";"AUTO AKTYWACJA:" +"KW-P04-33";"BŁĘDNE DANE" +"KW-P05-01";"ROZPOZNAWANIE TWARZY" +"KW-P05-02";"ROZPOZNAWANIE TWARZY:" +"KW-P05-04";"MODEL:" +"KW-P05-05";"Wykrycie obecności:" +"KW-P05-06";"WŁĄCZONY" +"KW-P05-07";"WYŁĄCZONY" +"KW-P05-08";"PODOBIEŃSTWO:" +"KW-P05-09";"NISKIE" +"KW-P05-10";"ŚREDNIE" +"KW-P05-11";"WYSOKIE" +"KW-P05-12";"ZAPISZ" +"KW-P06-01";"USTAWIENIA KAMER IP" +"KW-P06-02";"ILOŚĆ KAMER:" +"KW-P06-03";"KAMERA" +"KW-P06-04";"URL:" +"KW-P06-05";"ZAPISZ" +"KW-P07-01";"USTAWIENIA SIP" +"KW-P07-02";"WŁĄCZ SIP:" +"KW-P07-03";"SPRAWDŹ STATUS SIP" +"KW-P07-04";"SIP ZAREJESTROWANY" +"KW-P07-05";"BŁĄD REJESTRACJI SIP" +"KW-P07-06";"SERWER SIP:" +"KW-P07-07";"DOMENA:" +"KW-P07-08";"OUTBOUND:" +"KW-P07-09";"STUN IP:" +"KW-P07-10";"PORT STUN:" +"KW-P07-11";"H.264:" +"KW-P07-12";"UŻYTKOWNIK SIP:" +"KW-P07-13";"HASŁO SIP:" +"KW-P07-14";"CZAS ROZMOWY:" +"KW-P07-15";"CZAS DZWONIENIA:" +"KW-P07-16";"ZAPISZ" +"KW-P08-01";"USTAWIENIA SIP TRUNK" +"KW-P08-02";"WŁĄCZ SIP TRUNK:" +"KW-P08-03";"URL:" +"KW-P08-04";"ZAPISZ" +"KW-P09-01";"USTAWIENIA PRZEKIEROWAŃ" +"KW-P09-02";"IMPORT" +"KW-P09-03";"EKSPORT" +"KW-P09-04";"APARTAMENT" +"KW-P09-05";"NUMER" +"KW-P10-01";"USTAWIENIA ZAAWANSOWANE" +"KW-P10-02";"SZYBKIE WYBIERANIE:" +"KW-P10-03";"URL:" +"KW-P10-04";"ONU:" +"KW-P10-05";"MAPOWANIE POŁĄCZEŃ:" +"KW-P10-06";"BIAŁA LISTA:" +"KW-P10-07";"Lista telefoniczna:" +"KW-P10-08";"IMPORT" +"KW-P10-09";"EKSPORT" +"KW-P10-10";"IMPORTUJ" +"KW-P10-11";"WYSYŁANIE ZAKOŃCZONE" +"KW-P10-12";"UŻYJ WŁAŚCIWEGO PLIKU CSV." +"KW-P10-13";"OK" +"KW-P10-14";"ZAPISZ" +"KW-P11-01";"USTAWIENIA KODU PIN" +"KW-P11-02";"OBECNY PIN:" +"KW-P11-03";"NOWY PIN:" +"KW-P11-04";"POTWIERDŹ PIN:" +"KW-P11-05";"ZAPISZ" +"KW-P12-01";"WECHAT QR" +"KW-P12-02";"WŁĄCZ" +"KW-P12-03";"UUID:" +"KW-P12-04";"HASŁO:" +"KW-P12-05";"SERWER:" +"KW-P12-06";"WŁĄCZ CZYTNIK QR:" +"KW-P12-07";"STATUS:" +"KW-P12-08";"REJESTRACJA POMYŚLNIE" +"KW-P12-09";"REJESTRACJA NIE POWIODŁA SIĘ" +"KW-P12-10";"ZAPISZ" +"KW-P13-01";"PRZYWRACAĆ" +"KW-P13-02";"PRZYWRÓCIĆ USTAWIENIA FABRYCZNE" +"KW-P13-03";"POTWIERDZAĆ PRZYWRÓĆ USTAWIENIA FABRYCZNE?" +"KW-P13-04";"URZĄDZENIE REBOOT" diff --git a/tools/retro-library/charset_normalizer/data/sample-russian-2.txt b/tools/retro-library/charset_normalizer/data/sample-russian-2.txt new file mode 100644 index 000000000..79cf59863 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-russian-2.txt @@ -0,0 +1,5 @@ +В гимназии он не был в числе первых учеников (исключение составляли математика и латынь). Укоренившаяся система механического заучивания материала учащимися (которая, как он считал, наносит вред самому духу учёбы и творческому мышлению), а также авторитарное отношение учителей к ученикам вызывало у Альберта Эйнштейна неприятие, поэтому он часто вступал в споры со своими преподавателями. + +После окончательного разорения отца семейства в 1894 году Эйнштейны переехали из Мюнхена в итальянский город Павию, близ Милана. Сам Альберт оставался в Мюнхене ещё некоторое время, чтобы окончить все шесть классов гимназии. Так и не получив аттестата зрелости, в 1895 году он присоединился к своей семье в Милане. + +Осенью 1895 г. Альберт Эйнштейн прибыл в Швейцарию, чтобы сдать вступительные экзамены в Высшее техническое училище (Политехникум) в Цюрихе и стать преподавателем физики. Блестяще проявив себя на экзамене по математике, он в то же время провалил экзамены по ботанике и французскому языку, что не позволило ему поступить в Цюрихский Политехникум. Однако директор училища посоветовал молодому человеку поступить в выпускной класс школы в Аарау (Швейцария), чтобы получить аттестат и повторить поступление. diff --git a/tools/retro-library/charset_normalizer/data/sample-russian-3.txt b/tools/retro-library/charset_normalizer/data/sample-russian-3.txt new file mode 100644 index 000000000..fbf2db067 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-russian-3.txt @@ -0,0 +1,7 @@ +Москва́ (произношение (инф.)) — столица России, город федерального значения, административный центр Центрального федерального округа и центр Московской области, в состав которой не входит[6]. Крупнейший по численности населения город России и её субъект — 12 655 050[3] человек (2021), самый населённый из городов, полностью расположенных в Европе, занимает 22 место среди городов мира по численности населения[7], крупнейший русскоязычный город в мире. Центр Московской городской агломерации. + +Историческая столица Великого княжества Московского, Русского царства, Российской империи (в 1728—1732 годах[8][9][10][11]), Советской России и СССР. Город-герой. В Москве находятся федеральные органы государственной власти Российской Федерации (за исключением Конституционного суда), посольства иностранных государств, штаб-квартиры большинства крупнейших российских коммерческих организаций и общественных объединений. + +Расположена на западе России, на реке Москве в центре Восточно-Европейской равнины, в междуречье Оки и Волги. Как субъект федерации, Москва граничит с Московской и Калужской областями. + +Москва — популярный туристический центр России. Кремль, Красная площадь, Новодевичий монастырь и Церковь Вознесения в Коломенском входят в список объектов всемирного наследия ЮНЕСКО[12]. Она является важнейшим транспортным узлом: город обслуживают 6 аэропортов, 10 железнодорожных вокзалов, 3 речных порта (имеется речное сообщение с морями бассейнов Атлантического и Северного Ледовитого океанов). С 1935 года в Москве работает метрополитен. Москва — спортивный центр страны. В 1980 году в Москве прошли XXII летние Олимпийские игры, а в 2018 город стал одним из хозяев чемпионата мира по футболу. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/data/sample-russian.txt b/tools/retro-library/charset_normalizer/data/sample-russian.txt new file mode 100644 index 000000000..5a5e77c0b --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-russian.txt @@ -0,0 +1,5 @@ + ( ). (, , ), , . + + 1894 , . , . , 1895 . + + 1895 . , () . , , . (), . diff --git a/tools/retro-library/charset_normalizer/data/sample-spanish.txt b/tools/retro-library/charset_normalizer/data/sample-spanish.txt new file mode 100755 index 000000000..9b9aadcc8 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-spanish.txt @@ -0,0 +1,33 @@ +La creación + +1 En el principio creó Dios los cielos y la tierra. 2 Y la tierra estaba sin orden y vacía[a], y las tinieblas cubrían la superficie[b] del abismo, y el Espíritu de Dios se movía sobre la superficie[c] de las aguas. 3 Entonces dijo Dios: Sea la luz. Y hubo luz. 4 Y vio Dios que la luz era buena; y separó Dios la luz de las tinieblas. 5 Y llamó Dios a la luz día, y a las tinieblas llamó noche. Y fue la tarde y fue la mañana: un día. + +6 Entonces dijo Dios: Haya expansión[d] en medio de las aguas, y separe las aguas de las aguas. 7 E hizo Dios la expansión, y separó las aguas que estaban debajo de la expansión de las aguas que estaban sobre la expansión. Y fue así. 8 Y llamó Dios a la expansión cielos. Y fue la tarde y fue la mañana: el segundo día. + +9 Entonces dijo Dios: Júntense en un lugar las aguas que están debajo de los cielos, y que aparezca lo seco. Y fue así. 10 Y llamó Dios a lo seco tierra, y al conjunto de las aguas llamó mares. Y vio Dios que era bueno. 11 Y dijo Dios: Produzca la tierra vegetación[e]: hierbas[f] que den semilla, y árboles frutales que den fruto sobre la tierra según su género[g], con su semilla en él. Y fue así. 12 Y produjo la tierra vegetación[h]: hierbas[i] que dan semilla según su género, y árboles que dan fruto con su semilla en él, según su género. Y vio Dios que era bueno. 13 Y fue la tarde y fue la mañana: el tercer día. + +14 Entonces dijo Dios: Haya lumbreras[j] en la expansión de los cielos para separar el día de la noche, y sean para señales y para estaciones y para días y para años; 15 y sean por luminarias en la expansión de los cielos para alumbrar sobre la tierra. Y fue así. 16 E hizo Dios las dos grandes lumbreras[k], la lumbrera[l] mayor para dominio del día y la lumbrera[m] menor para dominio de la noche; hizo también las estrellas. 17 Y Dios las puso en la expansión de los cielos para alumbrar sobre la tierra, 18 y para dominar en el día y en la noche, y para separar la luz de las tinieblas. Y vio Dios que era bueno. 19 Y fue la tarde y fue la mañana: el cuarto día. + +20 Entonces dijo Dios: Llénense[n] las aguas de multitudes de seres vivientes, y vuelen las aves sobre la tierra en la abierta[o] expansión de los cielos. 21 Y creó Dios los grandes monstruos marinos y todo ser viviente que se mueve, de los cuales están llenas[p] las aguas según su género, y toda ave[q] según su género. Y vio Dios que era bueno. 22 Y Dios los bendijo, diciendo: Sed fecundos y multiplicaos, y llenad las aguas en los mares, y multiplíquense las aves en la tierra. 23 Y fue la tarde y fue la mañana: el quinto día. + +24 Entonces dijo Dios: Produzca la tierra seres vivientes según su género: ganados, reptiles y bestias de la tierra según su género. Y fue así. 25 E hizo Dios las bestias de la tierra según su género, y el ganado según su género, y todo lo que se arrastra sobre la tierra según su género. Y vio Dios que era bueno. +Creación del hombre y de la mujer + +26 Y dijo Dios: Hagamos al hombre a nuestra imagen, conforme a nuestra semejanza; y ejerza[r] dominio sobre los peces del mar, sobre las aves del cielo, sobre los ganados, sobre toda la tierra, y sobre todo reptil que se arrastra sobre la tierra. 27 Creó, pues, Dios al hombre a imagen suya, a imagen de Dios lo creó; varón y hembra los creó. 28 Y los bendijo Dios y les dijo[s]: Sed fecundos y multiplicaos, y llenad la tierra y sojuzgadla; ejerced dominio sobre los peces del mar, sobre las aves del cielo y sobre todo ser viviente que se mueve[t] sobre la tierra. 29 Y dijo Dios: He aquí, yo os he dado toda planta que da semilla que hay en la superficie[u] de toda la tierra, y todo árbol que tiene fruto[v] que da semilla; esto os servirá de[w] alimento. 30 Y a toda bestia de la tierra, a toda ave de los cielos y a todo lo que se mueve[x] sobre la tierra, y que tiene vida[y], les he dado toda planta verde para alimento. Y fue así. 31 Y vio Dios todo lo que había hecho, y he aquí que era bueno en gran manera. Y fue la tarde y fue la mañana: el sexto día. + +Así fueron acabados los cielos y la tierra y todas sus huestes. 2 Y en el séptimo día completó Dios la[a] obra que había hecho, y reposó en el día séptimo de toda la[b] obra que había hecho. 3 Y bendijo Dios el séptimo día y lo santificó, porque en él reposó de toda la[c] obra que El[d] había creado y hecho[e]. +El huerto del Edén + +4 Estos son los orígenes[f] de los cielos y de la tierra cuando fueron creados, el día en que el Señor Dios hizo la tierra y los cielos. 5 Y aún no había ningún arbusto del campo en la tierra, ni había aún brotado ninguna planta[g] del campo, porque el Señor Dios no había enviado lluvia sobre la tierra, ni había hombre para labrar[h] la tierra. 6 Pero se levantaba de la tierra un vapor[i] que regaba toda la superficie[j] del suelo. 7 Entonces el Señor Dios formó al hombre del polvo de la tierra, y sopló en su nariz el aliento de vida; y fue el hombre un ser[k] viviente. 8 Y plantó el Señor Dios un huerto hacia el oriente, en Edén; y puso allí al hombre que había formado. 9 Y el Señor Dios hizo brotar de la tierra todo árbol agradable a la vista y bueno para comer; asimismo, en medio del huerto, el árbol de la vida y el árbol del conocimiento[l] del bien y del mal. + +10 Y del Edén salía un río para regar el huerto, y de allí se dividía y se convertía en otros cuatro ríos[m]. 11 El nombre del primero es Pisón; éste es el que rodea toda la tierra de Havila, donde hay oro. 12 El oro de aquella tierra es bueno; allí hay bedelio y ónice. 13 Y el nombre del segundo río es Gihón; éste es el que rodea la tierra de Cus. 14 Y el nombre del tercer río es Tigris[n]; éste es el que corre[o] al oriente de Asiria. Y el cuarto río es el Eufrates[p]. 15 Entonces el Señor Dios tomó al hombre y lo puso en el huerto del Edén, para que lo cultivara y lo cuidara. 16 Y ordenó el Señor Dios al hombre, diciendo: De todo árbol del huerto podrás comer, 17 pero del árbol del conocimiento[q] del bien y del mal no comerás[r], porque el día que de él comas, ciertamente morirás. +Formación de la mujer + +18 Y el Señor Dios dijo: No es bueno que el hombre esté solo; le haré una ayuda idónea[s]. 19 Y el Señor Dios formó de la tierra todo animal del campo y toda ave del cielo, y los trajo al hombre para ver cómo los llamaría; y como el hombre llamó a cada ser viviente, ése fue su nombre. 20 Y el hombre puso nombre a todo ganado y a las aves del cielo y a toda bestia del campo, mas para Adán[t] no se encontró una ayuda que fuera idónea para él[u]. 21 Entonces el Señor Dios hizo caer un sueño profundo sobre el hombre, y éste se durmió; y Dios tomó una de sus costillas, y cerró la carne en ese lugar. 22 Y de la costilla que el Señor Dios había tomado del hombre, formó[v] una mujer y la trajo al hombre. 23 Y el hombre dijo: + +Esta es ahora hueso de mis huesos, +y carne de mi carne; +ella[w] será llamada mujer[x], +porque del hombre[y] fue tomada. + +24 Por tanto el hombre dejará a su padre y a su madre y se unirá a su mujer, y serán una sola carne. 25 Y estaban ambos desnudos, el hombre y su mujer, y no se avergonzaban. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/data/sample-turkish.txt b/tools/retro-library/charset_normalizer/data/sample-turkish.txt new file mode 100644 index 000000000..c86234301 --- /dev/null +++ b/tools/retro-library/charset_normalizer/data/sample-turkish.txt @@ -0,0 +1,33 @@ +stanbul, Trkiye'nin en kalabalk, iktisadi ve kltrel adan en nemli +ehri.[2][3][4] ktisadi byklk adan dnyada 34., nfus asndan +belediye snrlar gz nne alnarak yaplan sralamaya gre Avrupa'da +birinci srada gelir.[5][6] + + +stanbul Trkiye'nin kuzeybatsnda, Marmara kys ve Boazii boyunca, +Hali'i de evreleyecek ekilde kurulmutur.[7] stanbul ktalararas bir +ehir olup, Avrupa'daki blmne Avrupa Yakas veya Rumeli Yakas, +Asya'daki blmne ise Anadolu Yakas denir. Tarihte ilk olarak taraf +Marmara Denizi, Boazii ve Hali'in sard bir yarm ada zerinde kurulan +stanbul'un batdaki snrn stanbul Surlar oluturmaktayd. Gelime ve +byme srecinde surlarn her seferinde daha batya ilerletilerek ina +edilmesiyle 4 defa geniletilen ehrin [8] 39 ilesi vardr.[9] Snrlar +ierisinde ise bykehir belediyesi ile birlikte toplam 40 belediye +bulunmaktadr. + + +Dnyann en eski ehirlerinden biri olan stanbul, M.S. 330 - 395 yllar +arasnda Roma mparatorluu, 395 - 1204 ile 1261 - 1453 yllar arasnda +Dou Roma mparatorluu, 1204 - 1261 arasnda Latin mparatorluu ve son +olarak 1453 - 1922 yllar arasnda Osmanl mparatorluu'na bakentlik +yapt.[10] Ayrca, hilafetin Osmanl mparatorluu'na getii 1517'den, +kaldrld 1924'e kadar, stanbul slamiyet'in de merkezi oldu.[11] + +1453 ylnda fetihten sonra, kent Osmanl mparatorluu'nun drdnc +bakenti ilan edilidi ve Kostantiniyye Osmanl mparatorluu tarafndan +kentin resmi ad olarak kullanld ve 1923 ylnda Osmanl +mparatorluunun kne kadar, ou zaman bu ad kullanmda +kald. rnein Osmanl mparatorluu ve mahkemeleri, Kostantiniyye'de +yaymlanan resmi belgelerin kaynan belirtmek iin, "be-Makam- +Dar's-Saltanat- Kostantiniyyet'l-Mahrust'l-Mahmiyye" gibi balklar +kullanlrd.[17] diff --git a/tools/retro-library/charset_normalizer/dev-requirements.txt b/tools/retro-library/charset_normalizer/dev-requirements.txt new file mode 100644 index 000000000..325adac25 --- /dev/null +++ b/tools/retro-library/charset_normalizer/dev-requirements.txt @@ -0,0 +1,7 @@ +chardet==5.1.0 +pytest-cov==4.1.0 +Flask==2.2.3 +pytest>=7.4.4,<=8.3.3 +requests==2.31.0 +pre-commit +build diff --git a/tools/retro-library/charset_normalizer/docs/Makefile b/tools/retro-library/charset_normalizer/docs/Makefile new file mode 100755 index 000000000..b6888c162 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = Charset Normalizer +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/docs/api.rst b/tools/retro-library/charset_normalizer/docs/api.rst new file mode 100644 index 000000000..3a6fb1325 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/api.rst @@ -0,0 +1,100 @@ +.. _api: + +Developer Interfaces +==================== + +.. module:: charset_normalizer + +Main Interfaces +--------------- + +Those functions are publicly exposed and are protected through our BC guarantee. + +.. autofunction:: from_bytes +.. autofunction:: from_fp +.. autofunction:: from_path +.. autofunction:: is_binary + +.. autoclass:: charset_normalizer.models.CharsetMatches + :inherited-members: +.. autoclass:: charset_normalizer.models.CharsetMatch + :inherited-members: + +.. autofunction:: detect + +.. autofunction:: charset_normalizer.utils.set_logging_handler + + +Mess Detector +------------- + +.. autofunction:: charset_normalizer.md.mess_ratio + +This library allows you to extend the capabilities of the mess detector by extending the +class `MessDetectorPlugin`. + +.. autoclass:: charset_normalizer.md.MessDetectorPlugin + :inherited-members: + + +.. autofunction:: charset_normalizer.md.is_suspiciously_successive_range + + +Coherence Detector +------------------ + +.. autofunction:: charset_normalizer.cd.coherence_ratio + + +Utilities +--------- + +Some reusable functions used across the project. We do not guarantee the BC in this area. + +.. autofunction:: charset_normalizer.utils.is_accentuated + +.. autofunction:: charset_normalizer.utils.remove_accent + +.. autofunction:: charset_normalizer.utils.unicode_range + +.. autofunction:: charset_normalizer.utils.is_latin + +.. autofunction:: charset_normalizer.utils.is_punctuation + +.. autofunction:: charset_normalizer.utils.is_symbol + +.. autofunction:: charset_normalizer.utils.is_emoticon + +.. autofunction:: charset_normalizer.utils.is_separator + +.. autofunction:: charset_normalizer.utils.is_case_variable + +.. autofunction:: charset_normalizer.utils.is_cjk + +.. autofunction:: charset_normalizer.utils.is_hiragana + +.. autofunction:: charset_normalizer.utils.is_katakana + +.. autofunction:: charset_normalizer.utils.is_hangul + +.. autofunction:: charset_normalizer.utils.is_thai + +.. autofunction:: charset_normalizer.utils.is_unicode_range_secondary + +.. autofunction:: charset_normalizer.utils.any_specified_encoding + +.. autofunction:: charset_normalizer.utils.is_multi_byte_encoding + +.. autofunction:: charset_normalizer.utils.identify_sig_or_bom + +.. autofunction:: charset_normalizer.utils.should_strip_sig_or_bom + +.. autofunction:: charset_normalizer.utils.iana_name + +.. autofunction:: charset_normalizer.utils.range_scan + +.. autofunction:: charset_normalizer.utils.is_cp_similar + + +.. class:: os.PathLike +.. class:: typing.BinaryIO diff --git a/tools/retro-library/charset_normalizer/docs/community/faq.rst b/tools/retro-library/charset_normalizer/docs/community/faq.rst new file mode 100644 index 000000000..d1deff4dd --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/community/faq.rst @@ -0,0 +1,75 @@ +Frequently asked questions +=========================== + + +Is UTF-8 everywhere already? +---------------------------- + +Not really, that is a dangerous assumption. Looking at https://w3techs.com/technologies/overview/character_encoding may +seem like encoding detection is a thing of the past but not really. Solo based on 33k websites, you will find +3,4k responses without predefined encoding. 1,8k websites were not UTF-8, merely half! + +This statistic (w3techs) does not offer any ponderation, so one should not read it as +"I have a 97 % chance of hitting UTF-8 content on HTML content". + +(2021 Top 1000 sites from 80 countries in the world according to Data for SEO) https://github.com/potiuk/test-charset-normalizer + +First of all, neither requests, chardet or charset-normalizer are dedicated to HTML content. +The detection concern every text document, like SubRip Subtitle files for instance. And by my own experiences, I never had +a single database using full utf-8, many translated subtitles are from another era and never updated. + +It is so hard to find any stats at all regarding this matter. Users' usages can be very dispersed, so making +assumptions are unwise. + +The real debate is to state if the detection is an HTTP client matter or not. That is more complicated and not my field. + +Some individuals keep insisting that the *whole* Internet is UTF-8 ready. Those are absolutely wrong and very Europe and North America-centered, +In my humble experience, the countries in the world are very disparate in this evolution. And the Internet is not just about HTML content. +Having a thorough analysis of this is very scary. + +Should I bother using detection? +-------------------------------- + +In the last resort, yes. You should use well-established standards, eg. predefined encoding, at all times. +When you are left with no clue, you may use the detector to produce a usable output as fast as possible. + +Is it backward-compatible with Chardet? +--------------------------------------- + +If you use the legacy `detect` function, +Then this change is mostly backward-compatible, exception of a thing: + +- This new library support way more code pages (x3) than its counterpart Chardet. +- Based on the 30-ich charsets that Chardet support, expect roughly 80% BC results + +We do not guarantee this BC exact percentage through time. May vary but not by much. + +Isn't it the same as Chardet? +----------------------------- + +The objective is the same, provide you with the best answer (charset) we can given any sequence of bytes. +The method actually differs. + +We do not "train" anything to build a probe for a specific encoding. In addition to finding any languages (intelligent +design) by some rudimentary statistics (character frequency ordering) we built a mess detector to assist the language +detection. + +Any code page supported by your cPython is supported by charset-normalizer! It is that simple, no need to update the +library. It is as generic as we could do. + +I can't build standalone executable +----------------------------------- + +If you are using ``pyinstaller``, ``py2exe`` or alike, you may be encountering this or close to: + + ModuleNotFoundError: No module named 'charset_normalizer.md__mypyc' + +Why? + +- Your package manager picked up a optimized (for speed purposes) wheel that match your architecture and operating system. +- Finally, the module ``charset_normalizer.md__mypyc`` is imported via binaries and can't be seen using your tool. + +How to remedy? + +If your bundler program support it, set up a hook that implicitly import the hidden module. +Otherwise, follow the guide on how to install the vanilla version of this package. (Section: *Optional speedup extension*) diff --git a/tools/retro-library/charset_normalizer/docs/community/featured.rst b/tools/retro-library/charset_normalizer/docs/community/featured.rst new file mode 100644 index 000000000..a704a0bc6 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/community/featured.rst @@ -0,0 +1,51 @@ +Featured projects +================= + +Did you liked how ``charset-normalizer`` perform? and its quality? +You may be interested in those other project maintained by the same authors. +We aim to serve the opensource community the best and as inclusively as we can, no matter +your level or opinions. + +Niquests +-------- + +Started as a simple though.. IE 11 has built-in HTTP/2 support while Requests 2.32 does not! + +Most of our programs that interact with HTTP server are built with ``requests`` and +we aren't likely to switch without a substantial effort. + +We just might die at any moment, no notice whatsoever, knowingly that as a Python developer, +we never interacted with a HTTP/2 over TCP or HTTP/3 over QUIC capable server in 2023... + +.. image:: https://dabuttonfactory.com/button.png?t=Get+Niquests+Now&f=Ubuntu-Bold&ts=26&tc=fff&hp=45&vp=20&c=11&bgt=unicolored&bgc=15d798&be=1 + :target: https://github.com/jawah/niquests + +It is a fork of ``requests`` and no breaking changes are to be expected. We made sure that +your migration is effortless and safe. + +httpie-next +----------- + +Easy solution are cool, let us introduce you to HTTPie but with built-in support +for HTTP/2 and HTTP/3. +It is made available as a plugin, no effort required beside installing the said plugin. + +Enjoy HTTPie refreshed! + +.. image:: https://dabuttonfactory.com/button.png?t=Get+HTTPie-Next+Now&f=Ubuntu-Bold&ts=26&tc=fff&hp=45&vp=20&c=11&bgt=unicolored&bgc=15d798&be=1 + :target: https://github.com/Ousret/httpie-next + +Wassima +------- + +Did you ever wonder how would it feel like to leave the headache with root CAs (certificate authority)? +Well, you may, starting today, use your operating system trusted root CAs to verify +peer certificates with the at most comfort. + +It is enabled by default in Niquests, but you can use that awesome feature by itself. + +.. image:: https://dabuttonfactory.com/button.png?t=OS+root+CAs+for+Python&f=Ubuntu-Bold&ts=26&tc=fff&hp=45&vp=20&c=11&bgt=unicolored&bgc=15d798&be=1 + :target: https://github.com/jawah/wassima + +The solution is universal and served for (almost) every possible case. +You may remove the certifi package, let it rest in peace. diff --git a/tools/retro-library/charset_normalizer/docs/community/speedup.rst b/tools/retro-library/charset_normalizer/docs/community/speedup.rst new file mode 100644 index 000000000..1e65379ac --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/community/speedup.rst @@ -0,0 +1,50 @@ +Optional speedup extension +========================== + +Why? +---- + +charset-normalizer will always remain pure Python, meaning that a environment without any build capabilities will +run this program without any additional requirements. + +Nonetheless, starting from the version 3.0 we introduce and publish some platform specific wheels including a +pre-built extension. + +Most of the time is spent in the module `md.py` so we decided to "compile it" using Mypyc. + +(1) It does not require to have a separate code base +(2) Our project code base is rather simple and lightweight +(3) Mypyc is robust enough today +(4) Four times faster! + +How? +---- + +If your platform and/or architecture is not served by this swift optimization you may compile it easily yourself. +Following those instructions (provided you have the necessary toolchain installed): + + :: + + export CHARSET_NORMALIZER_USE_MYPYC=1 + pip install mypy build wheel + pip install charset-normalizer --no-binary :all: + + +How not to? +----------- + +You may install charset-normalizer without the speedups by directly using the universal wheel +(most likely hosted on PyPI or any valid mirror you use) with ``--no-binary``. + +E.g. when installing ``requests`` and you don't want to use the ``charset-normalizer`` speedups, you can do: + + :: + + pip install requests --no-binary charset-normalizer + + +When installing `charset-normalizer` by itself, you can also pass ``:all:`` as the specifier to ``--no-binary``. + + :: + + pip install charset-normalizer --no-binary :all: diff --git a/tools/retro-library/charset_normalizer/docs/community/why_migrate.rst b/tools/retro-library/charset_normalizer/docs/community/why_migrate.rst new file mode 100644 index 000000000..1909c7705 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/community/why_migrate.rst @@ -0,0 +1,18 @@ +Why should I migrate to Charset-Normalizer? +=========================================== + +There is so many reason to migrate your current project. Here are some of them: + +- Remove ANY license ambiguity/restriction for projects bundling Chardet (even indirectly). +- X10 faster than Chardet in average and X6 faster in 99% of the cases AND support 3 times more encoding. +- Never return a encoding if not suited for the given decoder. Eg. Never get UnicodeDecodeError! +- Actively maintained, open to contributors. +- Have the backward compatible function ``detect`` that come from Chardet. +- Truly detect the language used in the text. +- It is, for the first time, really universal! As there is no specific probe per charset. +- The package size is X2~X4 lower than Chardet's (5.0)! (Depends on your arch) +- Propose much more options/public kwargs to tweak the detection as you sees fit! +- Using static typing to ease your development. +- Detect Unicode content better than Chardet or cChardet does. + +And much more..! What are you waiting for? Upgrade now and give us a feedback. (Even if negative) diff --git a/tools/retro-library/charset_normalizer/docs/conf.py b/tools/retro-library/charset_normalizer/docs/conf.py new file mode 100755 index 000000000..46510fade --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/conf.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# charset-normalizer documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 16 04:30:35 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +import sys +import os + +sys.path.insert(0, os.path.abspath("..")) + +import charset_normalizer + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +# source_suffix = '.rst' + +source_parsers = {} + +source_suffix = ['.rst',] + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'charset_normalizer' +copyright = '2023, Ahmed TAHRI' +author = 'Ahmed TAHRI' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = charset_normalizer.__version__ +# The full version, including alpha/beta/rc tags. +release = charset_normalizer.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'furo' + +html_theme_path = [] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'charset-normalizer-doc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'charset-normalizer.tex', 'Charset Normalizer Documentation', + 'Ahmed TAHRI', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'charset-normalizer', 'Charset Normalizer Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +texinfo_documents = [ + (master_doc, 'Charset Normalizer', 'Charsert Normalizer Documentation', + author, 'charset-normalizer', '🔎 Like Chardet. 🚀 Package for encoding & language detection. Charset detection.', + 'Miscellaneous'), +] diff --git a/tools/retro-library/charset_normalizer/docs/index.rst b/tools/retro-library/charset_normalizer/docs/index.rst new file mode 100755 index 000000000..19ca08a90 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/index.rst @@ -0,0 +1,93 @@ +=================== + Charset Normalizer +=================== + +Overview +======== + +A Library that helps you read text from unknown charset encoding. +This project is motivated by chardet, I'm trying to resolve the issue by taking another approach. +All IANA character set names for which the Python core library provides codecs are supported. + +It aims to be as generic as possible. + +.. image:: https://repository-images.githubusercontent.com/200259335/d3da9600-dedc-11e9-83e8-081f597505df + :width: 500px + :alt: CLI Charset Normalizer + :align: right + + +It is released under MIT license, see LICENSE for more +details. Be aware that no warranty of any kind is provided with this package. + +Copyright (C) 2023 Ahmed TAHRI + +Introduction +============ + +This library aim to assist you in finding what encoding suit the best to content. +It **DOES NOT** try to uncover the originating encoding, in fact this program does not care about it. + +By originating we means the one that was precisely used to encode a text file. + +Precisely :: + + my_byte_str = 'Bonjour, je suis à la recherche d\'une aide sur les étoiles'.encode('cp1252') + + +We **ARE NOT** looking for cp1252 **BUT FOR** ``Bonjour, je suis à la recherche d'une aide sur les étoiles``. +Because of this :: + + my_byte_str.decode('cp1252') == my_byte_str.decode('cp1256') == my_byte_str.decode('cp1258') == my_byte_str.decode('iso8859_14') + # Print True ! + +There is no wrong answer to decode ``my_byte_str`` to get the exact same result. +This is where this library differ from others. There's not specific probe per encoding table. + +Features +======== + +- Encoding detection on a fp (file pointer), bytes or PathLike. +- Transpose any encoded content to Unicode the best we can. +- Detect spoken language in text. +- Ship with a great CLI. +- Also, detect binaries. + +Start Guide +----------- + +.. toctree:: + :maxdepth: 2 + + user/support + user/getstarted + user/advanced_search + user/handling_result + user/miscellaneous + user/cli + +Community Guide +--------------- + +.. toctree:: + :maxdepth: 2 + + community/speedup + community/faq + community/why_migrate + community/featured + +Developer Guide +--------------- + +.. toctree:: + :maxdepth: 3 + + api + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tools/retro-library/charset_normalizer/docs/make.bat b/tools/retro-library/charset_normalizer/docs/make.bat new file mode 100644 index 000000000..4be663f46 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=build +set SPHINXPROJ=charset_normalizer + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/tools/retro-library/charset_normalizer/docs/requirements.txt b/tools/retro-library/charset_normalizer/docs/requirements.txt new file mode 100755 index 000000000..de8fca92b --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/requirements.txt @@ -0,0 +1,2 @@ +Sphinx +furo diff --git a/tools/retro-library/charset_normalizer/docs/user/advanced_search.rst b/tools/retro-library/charset_normalizer/docs/user/advanced_search.rst new file mode 100644 index 000000000..a269cd102 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/user/advanced_search.rst @@ -0,0 +1,82 @@ +Advanced Search +=============== + +Charset Normalizer method ``from_bytes``, ``from_fp`` and ``from_path`` provide some +optional parameters that can be tweaked. + +As follow :: + + from charset_normalizer import from_bytes + + my_byte_str = 'Bсеки човек има право на образование.'.encode('cp1251') + + results = from_bytes( + my_byte_str, + steps=10, # Number of steps/block to extract from my_byte_str + chunk_size=512, # Set block size of each extraction + threshold=0.2, # Maximum amount of chaos allowed on first pass + cp_isolation=None, # Finite list of encoding to use when searching for a match + cp_exclusion=None, # Finite list of encoding to avoid when searching for a match + preemptive_behaviour=True, # Determine if we should look into my_byte_str (ASCII-Mode) for pre-defined encoding + explain=False, # Print on screen what is happening when searching for a match + language_threshold=0.1 # Minimum coherence ratio / language ratio match accepted + ) + + +Using CharsetMatches +------------------------------ + +Here, ``results`` is a ``CharsetMatches`` object. It behave like a list but does not implements all related methods. +Initially, it is sorted. Calling ``best()`` is sufficient to extract the most probable result. + +.. autoclass:: charset_normalizer.CharsetMatches + :members: + +List behaviour +-------------- + +Like said earlier, ``CharsetMatches`` object behave like a list. + + :: + + # Call len on results also work + if not results: + print('No match for your sequence') + + # Iterate over results like a list + for match in results: + print(match.encoding, 'can decode properly your sequence using', match.alphabets, 'and language', match.language) + + # Using index to access results + if results: + print(str(results[0])) + +Using best() +------------ + +Like said above, ``CharsetMatches`` object behave like a list and it is sorted by default after getting results from +``from_bytes``, ``from_fp`` or ``from_path``. + +Using ``best()`` return the most probable result, the first entry of the list. Eg. idx 0. +It return a ``CharsetMatch`` object as return value or None if there is not results inside it. + + :: + + result = results.best() + +Calling first() +--------------- + +The very same thing than calling the method ``best()``. + +Class aliases +------------- + +``CharsetMatches`` is also known as ``CharsetDetector``, ``CharsetDoctor`` and ``CharsetNormalizerMatches``. +It is useful if you prefer short class name. + +Verbose output +-------------- + +You may want to understand why a specific encoding was not picked by charset_normalizer. All you have to do is passing +``explain`` to True when using methods ``from_bytes``, ``from_fp`` or ``from_path``. diff --git a/tools/retro-library/charset_normalizer/docs/user/cli.rst b/tools/retro-library/charset_normalizer/docs/user/cli.rst new file mode 100644 index 000000000..6dba8b489 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/user/cli.rst @@ -0,0 +1,116 @@ +Command Line Interface +====================== + +charset-normalizer ship with a CLI that should be available as `normalizer`. +This is a great tool to fully exploit the detector capabilities without having to write Python code. + +Possible use cases: + +#. Quickly discover probable originating charset from a file. +#. I want to quickly convert a non Unicode file to Unicode. +#. Debug the charset-detector. + +Down below, we will guide you through some basic examples. + +Arguments +--------- + +You may simply invoke `normalizer -h` (with the h(elp) flag) to understand the basics. + +:: + + usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD] + file [file ...] + + The Real First Universal Charset Detector. Discover originating encoding used + on text file. Normalize text to unicode. + + positional arguments: + files File(s) to be analysed + + optional arguments: + -h, --help show this help message and exit + -v, --verbose Display complementary information about file if any. + Stdout will contain logs about the detection process. + -a, --with-alternative + Output complementary possibilities if any. Top-level + JSON WILL be a list. + -n, --normalize Permit to normalize input file. If not set, program + does not write anything. + -m, --minimal Only output the charset detected to STDOUT. Disabling + JSON output. + -r, --replace Replace file when trying to normalize it instead of + creating a new one. + -f, --force Replace file without asking if you are sure, use this + flag with caution. + -t THRESHOLD, --threshold THRESHOLD + Define a custom maximum amount of chaos allowed in + decoded content. 0. <= chaos <= 1. + --version Show version information and exit. + +.. code:: bash + + normalizer ./data/sample.1.fr.srt + +You may also run the command line interface using: + +.. code:: bash + + python -m charset_normalizer ./data/sample.1.fr.srt + + +Main JSON Output +---------------- + +🎉 Since version 1.4.0 the CLI produce easily usable stdout result in +JSON format. + +.. code:: json + + { + "path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt", + "encoding": "cp1252", + "encoding_aliases": [ + "1252", + "windows_1252" + ], + "alternative_encodings": [ + "cp1254", + "cp1256", + "cp1258", + "iso8859_14", + "iso8859_15", + "iso8859_16", + "iso8859_3", + "iso8859_9", + "latin_1", + "mbcs" + ], + "language": "French", + "alphabets": [ + "Basic Latin", + "Latin-1 Supplement" + ], + "has_sig_or_bom": false, + "chaos": 0.149, + "coherence": 97.152, + "unicode_path": null, + "is_preferred": true + } + + +I recommend the `jq` command line tool to easily parse and exploit specific data from the produced JSON. + +Multiple File Input +------------------- + +It is possible to give multiple files to the CLI. It will produce a list instead of an object at the top level. +When using the `-m` (minimal output) it will rather print one result (encoding) per line. + +Unicode Conversion +------------------ + +If you desire to convert any file to Unicode you will need to append the flag `-n`. It will produce another file, +it won't replace it by default. + +The newly created file path will be declared in `unicode_path` (JSON output). diff --git a/tools/retro-library/charset_normalizer/docs/user/getstarted.rst b/tools/retro-library/charset_normalizer/docs/user/getstarted.rst new file mode 100644 index 000000000..47fb2287d --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/user/getstarted.rst @@ -0,0 +1,73 @@ +Installation +============ + +This installs a package that can be used from Python (``import charset_normalizer``). + +To install for all users on the system, administrator rights (root) may be required. + +Using PIP +--------- +Charset Normalizer can be installed from pip:: + + pip install charset-normalizer + +You may retrieve the latest unicodedata backport as follow:: + + pip install charset-normalizer[unicode_backport] + +From git via master +----------------------- +You can install from dev-master branch using git:: + + git clone https://github.com/Ousret/charset_normalizer.git + cd charset_normalizer/ + python setup.py install + +Basic Usage +=========== + +The new way +----------- + +You may want to get right to it. :: + + from charset_normalizer import from_bytes, from_path + + # This is going to print out your sequence once properly decoded + print( + str( + from_bytes( + my_byte_str + ).best() + ) + ) + + # You could also want the same from a file + print( + str( + from_path( + './data/sample.1.ar.srt' + ).best() + ) + ) + + +Backward compatibility +---------------------- + +If you were used to python chardet, we are providing the very same ``detect()`` method as chardet. +This function is mostly backward-compatible with Chardet. The migration should be painless. + + :: + + from charset_normalizer import detect + + # This will behave exactly the same as python chardet + result = detect(my_byte_str) + + if result['encoding'] is not None: + print('got', result['encoding'], 'as detected encoding') + + +You may upgrade your code with ease. +CTRL + R ``from chardet import detect`` to ``from charset_normalizer import detect``. diff --git a/tools/retro-library/charset_normalizer/docs/user/handling_result.rst b/tools/retro-library/charset_normalizer/docs/user/handling_result.rst new file mode 100644 index 000000000..b4a6ac46a --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/user/handling_result.rst @@ -0,0 +1,25 @@ +================ + Handling Result +================ + +When initiating search upon a buffer, bytes or file you can assign the return value and fully exploit it. + + :: + + my_byte_str = 'Bсеки човек има право на образование.'.encode('cp1251') + + # Assign return value so we can fully exploit result + result = from_bytes( + my_byte_str + ).best() + + print(result.encoding) # cp1251 + +Using CharsetMatch +---------------------------- + +Here, ``result`` is a ``CharsetMatch`` object or ``None``. + +.. autoclass:: charset_normalizer.CharsetMatch + :members: + diff --git a/tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst b/tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst new file mode 100644 index 000000000..c6251396d --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst @@ -0,0 +1,64 @@ +============== + Miscellaneous +============== + +Convert to str +-------------- + +Any ``CharsetMatch`` object can be transformed to exploitable ``str`` variable. + + :: + + my_byte_str = 'Bсеки човек има право на образование.'.encode('cp1251') + + # Assign return value so we can fully exploit result + result = from_bytes( + my_byte_str + ).best() + + # This should print 'Bсеки човек има право на образование.' + print(str(result)) + + +Logging +------- + +Prior to the version 2.0.11 you may encounter some unexpected logs in your streams. +Something along the line of: + + :: + + ... | WARNING | override steps (5) and chunk_size (512) as content does not fit (465 byte(s) given) parameters. + ... | INFO | ascii passed initial chaos probing. Mean measured chaos is 0.000000 % + ... | INFO | ascii should target any language(s) of ['Latin Based'] + + +It is most likely because you altered the root getLogger instance. The package has its own logic behind logging and why +it is useful. See https://docs.python.org/3/howto/logging.html to learn the basics. + +If you are looking to silence and/or reduce drastically the amount of logs, please upgrade to the latest version +available for `charset-normalizer` using your package manager or by `pip install charset-normalizer -U`. + +The latest version will no longer produce any entry greater than `DEBUG`. +On `DEBUG` only one entry will be observed and that is about the detection result. + +Then regarding the others log entries, they will be pushed as `Level 5`. Commonly known as TRACE level, but we do +not register it globally. + + +Detect binaries +--------------- + +This package offers a neat way to detect files that can be considered as 'binaries' +meaning that it is not likely to be a text-file. + + :: + + from charset_normalizer import is_binary + + # It can receive both a path or bytes or even a file pointer. + result = is_binary("./my-file.ext") + + # This should print 'True' or 'False' + print(result) + diff --git a/tools/retro-library/charset_normalizer/docs/user/support.rst b/tools/retro-library/charset_normalizer/docs/user/support.rst new file mode 100644 index 000000000..b00aa9dc2 --- /dev/null +++ b/tools/retro-library/charset_normalizer/docs/user/support.rst @@ -0,0 +1,183 @@ +================= + Support +================= + +**If you are running:** + +- Python >=2.7,<3.5: Unsupported +- Python 3.5: charset-normalizer < 2.1 +- Python 3.6: charset-normalizer < 3.1 + +Upgrade your Python interpreter as soon as possible. + +------------------- +Supported Encodings +------------------- + +Here are a list of supported encoding and supported language with latest update. Also this list +may change depending of your python version. + +Charset Normalizer is able to detect any of those encoding. This list is NOT static and depends heavily on what your +current cPython version is shipped with. See https://docs.python.org/3/library/codecs.html#standard-encodings + +=============== =============================================================================================================================== +IANA Code Page Aliases +=============== =============================================================================================================================== +ascii 646, ansi_x3.4_1968, ansi_x3_4_1968, ansi_x3.4_1986, cp367, csascii, ibm367, iso646_us, iso_646.irv_1991, iso_ir_6, us, us_ascii +big5 big5_tw, csbig5, x_mac_trad_chinese +big5hkscs big5_hkscs, hkscs +cp037 037, csibm037, ebcdic_cp_ca, ebcdic_cp_nl, ebcdic_cp_us, ebcdic_cp_wt, ibm037, ibm039 +cp1026 1026, csibm1026, ibm1026 +cp1125 1125, ibm1125, cp866u, ruscii +cp1140 1140, ibm1140 +cp1250 1250, windows_1250 +cp1251 1251, windows_1251 +cp1252 1252, windows_1252 +cp1253 1253, windows_1253 +cp1254 1254, windows_1254 +cp1255 1255, windows_1255 +cp1256 1256, windows_1256 +cp1257 1257, windows_1257 +cp1258 1258, windows_1258 +cp273 273, ibm273, csibm273 +cp424 424, csibm424, ebcdic_cp_he, ibm424 +cp437 437, cspc8codepage437, ibm437 +cp500 500, csibm500, ebcdic_cp_be, ebcdic_cp_ch, ibm500 +cp775 775, cspc775baltic, ibm775 +cp850 850, cspc850multilingual, ibm850 +cp852 852, cspcp852, ibm852 +cp855 855, csibm855, ibm855 +cp857 857, csibm857, ibm857 +cp858 858, csibm858, ibm858 +cp860 860, csibm860, ibm860 +cp861 861, cp_is, csibm861, ibm861 +cp862 862, cspc862latinhebrew, ibm862 +cp863 863, csibm863, ibm863 +cp864 864, csibm864, ibm864 +cp865 865, csibm865, ibm865 +cp866 866, csibm866, ibm866 +cp869 869, cp_gr, csibm869, ibm869 +cp932 932, ms932, mskanji, ms_kanji +cp949 949, ms949, uhc +cp950 950, ms950 +euc_jis_2004 jisx0213, eucjis2004, euc_jis2004 +euc_jisx0213 eucjisx0213 +euc_jp eucjp, ujis, u_jis +euc_kr euckr, korean, ksc5601, ks_c_5601, ks_c_5601_1987, ksx1001, ks_x_1001, x_mac_korean +gb18030 gb18030_2000 +gb2312 chinese, csiso58gb231280, euc_cn, euccn, eucgb2312_cn, gb2312_1980, gb2312_80, iso_ir_58, x_mac_simp_chinese +gbk 936, cp936, ms936 +hp_roman8 roman8, r8, csHPRoman8 +hz hzgb, hz_gb, hz_gb_2312 +iso2022_jp csiso2022jp, iso2022jp, iso_2022_jp +iso2022_jp_1 iso2022jp_1, iso_2022_jp_1 +iso2022_jp_2 iso2022jp_2, iso_2022_jp_2 +iso2022_jp_3 iso2022jp_3, iso_2022_jp_3 +iso2022_jp_ext iso2022jp_ext, iso_2022_jp_ext +iso2022_kr csiso2022kr, iso2022kr, iso_2022_kr +iso8859_10 csisolatin6, iso_8859_10, iso_8859_10_1992, iso_ir_157, l6, latin6 +iso8859_11 thai, iso_8859_11, iso_8859_11_2001 +iso8859_13 iso_8859_13, l7, latin7 +iso8859_14 iso_8859_14, iso_8859_14_1998, iso_celtic, iso_ir_199, l8, latin8 +iso8859_15 iso_8859_15, l9, latin9 +iso8859_16 iso_8859_16, iso_8859_16_2001, iso_ir_226, l10, latin10 +iso8859_2 csisolatin2, iso_8859_2, iso_8859_2_1987, iso_ir_101, l2, latin2 +iso8859_3 csisolatin3, iso_8859_3, iso_8859_3_1988, iso_ir_109, l3, latin3 +iso8859_4 csisolatin4, iso_8859_4, iso_8859_4_1988, iso_ir_110, l4, latin4 +iso8859_5 csisolatincyrillic, cyrillic, iso_8859_5, iso_8859_5_1988, iso_ir_144 +iso8859_6 arabic, asmo_708, csisolatinarabic, ecma_114, iso_8859_6, iso_8859_6_1987, iso_ir_127 +iso8859_7 csisolatingreek, ecma_118, elot_928, greek, greek8, iso_8859_7, iso_8859_7_1987, iso_ir_126 +iso8859_8 csisolatinhebrew, hebrew, iso_8859_8, iso_8859_8_1988, iso_ir_138 +iso8859_9 csisolatin5, iso_8859_9, iso_8859_9_1989, iso_ir_148, l5, latin5 +iso2022_jp_2004 iso_2022_jp_2004, iso2022jp_2004 +johab cp1361, ms1361 +koi8_r cskoi8r +kz1048 kz_1048, rk1048, strk1048_2002 +latin_1 8859, cp819, csisolatin1, ibm819, iso8859, iso8859_1, iso_8859_1, iso_8859_1_1987, iso_ir_100, l1, latin, latin1 +mac_cyrillic maccyrillic +mac_greek macgreek +mac_iceland maciceland +mac_latin2 maccentraleurope, maclatin2 +mac_roman macintosh, macroman +mac_turkish macturkish +ptcp154 csptcp154, pt154, cp154, cyrillic_asian +shift_jis csshiftjis, shiftjis, sjis, s_jis, x_mac_japanese +shift_jis_2004 shiftjis2004, sjis_2004, s_jis_2004 +shift_jisx0213 shiftjisx0213, sjisx0213, s_jisx0213 +tis_620 tis620, tis_620_0, tis_620_2529_0, tis_620_2529_1, iso_ir_166 +utf_16 u16, utf16 +utf_16_be unicodebigunmarked, utf_16be +utf_16_le unicodelittleunmarked, utf_16le +utf_32 u32, utf32 +utf_32_be utf_32be +utf_32_le utf_32le +utf_8 u8, utf, utf8, utf8_ucs2, utf8_ucs4 (+utf_8_sig) +utf_7* u7, unicode-1-1-utf-7 +cp720 N.A. +cp737 N.A. +cp856 N.A. +cp874 N.A. +cp875 N.A. +cp1006 N.A. +koi8_r N.A. +koi8_t N.A. +koi8_u N.A. +=============== =============================================================================================================================== + +*: Only if a SIG/mark is found. + +------------------- +Supported Languages +------------------- + +Those language can be detected inside your content. All of these are specified in ./charset_normalizer/assets/__init__.py . + + +| English, +| German, +| French, +| Dutch, +| Italian, +| Polish, +| Spanish, +| Russian, +| Japanese, +| Portuguese, +| Swedish, +| Chinese, +| Ukrainian, +| Norwegian, +| Finnish, +| Vietnamese, +| Czech, +| Hungarian, +| Korean, +| Indonesian, +| Turkish, +| Romanian, +| Farsi, +| Arabic, +| Danish, +| Serbian, +| Lithuanian, +| Slovene, +| Slovak, +| Malay, +| Hebrew, +| Bulgarian, +| Croatian, +| Hindi, +| Estonian, +| Thai, +| Greek, +| Tamil. + +---------------------------- +Incomplete Sequence / Stream +---------------------------- + +It is not (yet) officially supported. If you feed an incomplete byte sequence (eg. truncated multi-byte sequence) the detector will +most likely fail to return a proper result. +If you are purposely feeding part of your payload for performance concerns, you may stop doing it as this package is fairly optimized. + +We are working on a dedicated way to handle streams. diff --git a/tools/retro-library/charset_normalizer/pyproject.toml b/tools/retro-library/charset_normalizer/pyproject.toml new file mode 100644 index 000000000..05f32e5ac --- /dev/null +++ b/tools/retro-library/charset_normalizer/pyproject.toml @@ -0,0 +1,81 @@ +[build-system] +requires = ["setuptools", "setuptools-scm", "mypy>=1.4.1,<=1.13.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "charset-normalizer" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +license = {text = "MIT"} +keywords = ["encoding", "charset", "charset-detector", "detector", "normalization", "unicode", "chardet", "detect"] +authors = [ + {name = "Ahmed R. TAHRI", email="tahri.ahmed@proton.me"}, +] +maintainers = [ + {name = "Ahmed R. TAHRI", email="tahri.ahmed@proton.me"}, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Text Processing :: Linguistic", + "Topic :: Utilities", + "Typing :: Typed", +] +requires-python = ">=3.7" +dynamic = ["version", "readme"] + +[project.optional-dependencies] +unicode_backport = [] + +[tool.setuptools.dynamic] +version = {attr = "charset_normalizer.__version__"} +readme = {file = ["README.md", "CHANGELOG.md", "LICENSE"]} + +[project.scripts] +normalizer = "charset_normalizer:cli.cli_detect" + +[project.urls] +"Changelog" = "https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md" +"Documentation" = "https://charset-normalizer.readthedocs.io/" +"Code" = "https://github.com/jawah/charset_normalizer" +"Issue tracker" = "https://github.com/jawah/charset_normalizer/issues" + +[tool.setuptools.packages.find] +exclude = ["tests*"] + +[tool.pytest.ini_options] +addopts = "--cov=charset_normalizer --cov-report=term-missing -rxXs" + +[tool.isort] +profile = "black" +add_imports = "from __future__ import annotations" + +[tool.mypy] +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_subclassing_any = true +disallow_untyped_calls = true +disallow_untyped_decorators = true +disallow_untyped_defs = true +no_implicit_optional = true +no_implicit_reexport = true +show_error_codes = true +strict_equality = true +warn_redundant_casts = true +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = false diff --git a/tools/retro-library/charset_normalizer/setup.cfg b/tools/retro-library/charset_normalizer/setup.cfg new file mode 100644 index 000000000..5d23e2539 --- /dev/null +++ b/tools/retro-library/charset_normalizer/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +ignore = W503, E203, B305 +max-line-length = 120 diff --git a/tools/retro-library/charset_normalizer/setup.py b/tools/retro-library/charset_normalizer/setup.py new file mode 100644 index 000000000..c113acda9 --- /dev/null +++ b/tools/retro-library/charset_normalizer/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +from __future__ import annotations + +import os +import sys + +from setuptools import setup + +USE_MYPYC = False + +if len(sys.argv) > 1 and sys.argv[1] == "--use-mypyc": + sys.argv.pop(1) + USE_MYPYC = True +if os.getenv("CHARSET_NORMALIZER_USE_MYPYC", None) == "1": + USE_MYPYC = True + +if USE_MYPYC: + from mypyc.build import mypycify + + MYPYC_MODULES = mypycify( + [ + "charset_normalizer/md.py", + ], + debug_level="0", + ) +else: + MYPYC_MODULES = None + +setup(name="charset-normalizer", ext_modules=MYPYC_MODULES) diff --git a/tools/retro-library/charset_normalizer/tests/__init__.py b/tools/retro-library/charset_normalizer/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/retro-library/charset_normalizer/tests/test_base_detection.py b/tools/retro-library/charset_normalizer/tests/test_base_detection.py new file mode 100644 index 000000000..e4fb5fd39 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_base_detection.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import pytest + +from charset_normalizer.api import from_bytes +from charset_normalizer.models import CharsetMatches + + +def test_empty(): + best_guess = from_bytes(b"").best() + + assert best_guess is not None, "Empty bytes payload SHOULD NOT return None" + assert ( + best_guess.encoding == "utf_8" + ), "Empty bytes payload SHOULD be guessed as UTF-8 (arbitrary)" + assert len(best_guess.alphabets) == 0, "" + + +def test_bool_matches(): + guesses_not_empty = from_bytes(b"") + guesses_empty = CharsetMatches([]) + + assert ( + bool(guesses_not_empty) is True + ), "Bool behaviour of CharsetMatches altered, should be True" + assert ( + bool(guesses_empty) is False + ), "Bool behaviour of CharsetMatches altered, should be False" + + +@pytest.mark.parametrize( + "payload, expected_encoding", + [ + (b"\xfe\xff", "utf_16"), + ("\uFEFF".encode("gb18030"), "gb18030"), + (b"\xef\xbb\xbf", "utf_8"), + ("".encode("utf_32"), "utf_32"), + ], +) +def test_empty_but_with_bom_or_sig(payload, expected_encoding): + best_guess = from_bytes(payload).best() + + assert best_guess is not None, "Empty detection but with SIG/BOM has failed!" + assert ( + best_guess.encoding == expected_encoding + ), "Empty detection but with SIG/BOM is wrongly detected!" + assert ( + best_guess.raw == payload + ), "The RAW property should contain the original payload given for detection." + assert best_guess.byte_order_mark is True, "The BOM/SIG property should return True" + assert str(best_guess) == "", "The cast to str SHOULD be empty" + + +@pytest.mark.parametrize( + "payload, expected_encoding", + [ + ( + ("\uFEFF" + "我没有埋怨，磋砣的只是一些时间。").encode("gb18030"), + "gb18030", + ), + ( + "我没有埋怨，磋砣的只是一些时间。".encode("utf_32"), + "utf_32", + ), + ( + "我没有埋怨，磋砣的只是一些时间。".encode("utf_8_sig"), + "utf_8", + ), + ], +) +def test_content_with_bom_or_sig(payload, expected_encoding): + best_guess = from_bytes(payload).best() + + assert best_guess is not None, "Detection but with SIG/BOM has failed!" + assert ( + best_guess.encoding == expected_encoding + ), "Detection but with SIG/BOM is wrongly detected!" + assert best_guess.byte_order_mark is True, "The BOM/SIG property should return True" + + +@pytest.mark.parametrize( + "payload", + [ + b"AbAdZ pOoooOlDl mmlDoDkA lldDkeEkddA mpAlkDF", + b"g4UsPJdfzNkGW2jwmKDGDilKGKYtpF2X.mx3MaTWL1tL7CNn5U7DeCcodKX7S3lwwJPKNjBT8etY", + b'{"token": "g4UsPJdfzNkGW2jwmKDGDilKGKYtpF2X.mx3MaTWL1tL7CNn5U7DeCcodKX7S3lwwJPKNjBT8etY"}', + b"81f4ab054b39cb0e12701e734077d84264308f5fc79494fc5f159fa2ebc07b73c8cc0e98e009664a20986706f90146e8eefcb929ce1f74a8eab21369fdc70198", + b"{}", + ], +) +def test_obviously_ascii_content(payload): + best_guess = from_bytes(payload).best() + + assert best_guess is not None, "Dead-simple ASCII detection has failed!" + assert ( + best_guess.encoding == "ascii" + ), "Dead-simple ASCII detection is wrongly detected!" + + +@pytest.mark.parametrize( + "payload", + [ + "\u020d\x1b".encode(), + "h\xe9llo world!\n".encode(), + "我没有埋怨，磋砣的只是一些时间。".encode(), + "Bсеки човек има право на образование. Oбразованието трябва да бъде безплатно, поне що се отнася до началното и основното образование.".encode(), + "Bсеки човек има право на образование.".encode(), + "(° ͜ʖ °), creepy face, smiley 😀".encode(), + """["Financiën", "La France"]""".encode(), + "Qu'est ce que une étoile?".encode(), + """Financiën""".encode(), + "😀".encode(), + ], +) +def test_obviously_utf8_content(payload): + best_guess = from_bytes(payload).best() + + assert best_guess is not None, "Dead-simple UTF-8 detection has failed!" + assert ( + best_guess.encoding == "utf_8" + ), "Dead-simple UTF-8 detection is wrongly detected!" + + +def test_mb_cutting_chk(): + # This payload should be wrongfully split and the autofix should ran automatically + # on chunks extraction. + payload = ( + b"\xbf\xaa\xbb\xe7\xc0\xfb \xbf\xb9\xbc\xf6 " + b" \xbf\xac\xb1\xb8\xc0\xda\xb5\xe9\xc0\xba \xba\xb9\xc0\xbd\xbc\xad\xb3\xaa " + * 128 + ) + + guesses = from_bytes(payload, cp_isolation=["cp949"]) + best_guess = guesses.best() + + assert len(guesses) == 1, "cp isolation is set and given seq should be clear CP949!" + assert best_guess.encoding == "cp949" + + +def test_alphabets_property(): + best_guess = from_bytes("😀 Hello World! How affairs are going? 😀".encode()).best() + + assert "Basic Latin" in best_guess.alphabets + assert "Emoticons range(Emoji)" in best_guess.alphabets + assert best_guess.alphabets.count("Basic Latin") == 1 + + +def test_doc_example_short_cp1251(): + best_guess = from_bytes( + "Bсеки човек има право на образование.".encode("cp1251") + ).best() + + assert best_guess.encoding == "cp1251" + + +def test_direct_cmp_charset_match(): + best_guess = from_bytes("😀 Hello World! How affairs are going? 😀".encode()).best() + + assert best_guess == "utf_8" + assert best_guess == "utf-8" + assert best_guess != 8 + assert best_guess != None diff --git a/tools/retro-library/charset_normalizer/tests/test_cli.py b/tools/retro-library/charset_normalizer/tests/test_cli.py new file mode 100644 index 000000000..5f2777c41 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_cli.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import unittest +from os import pardir, path, remove +from os.path import exists +from unittest.mock import patch + +from charset_normalizer.cli import cli_detect, query_yes_no + +DIR_PATH = path.join(path.dirname(path.realpath(__file__)), pardir) + + +class TestCommandLineInterface(unittest.TestCase): + @patch("builtins.input", lambda *args: "y") + def test_simple_yes_input(self): + self.assertTrue(query_yes_no("Are u willing to chill a little bit ?")) + + @patch("builtins.input", lambda *args: "N") + def test_simple_no_input(self): + self.assertFalse(query_yes_no("Are u willing to chill a little bit ?")) + + def test_single_file(self): + self.assertEqual(0, cli_detect([DIR_PATH + "/data/sample-arabic-1.txt"])) + + def test_version_output_success(self): + with self.assertRaises(SystemExit): + cli_detect(["--version"]) + + def test_single_file_normalize(self): + self.assertEqual( + 0, cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--normalize"]) + ) + + self.assertTrue(exists(DIR_PATH + "/data/sample-arabic-1.cp1256.txt")) + + try: + remove(DIR_PATH + "/data/sample-arabic-1.cp1256.txt") + except: + pass + + def test_single_verbose_file(self): + self.assertEqual( + 0, cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--verbose"]) + ) + + def test_multiple_file(self): + self.assertEqual( + 0, + cli_detect( + [ + DIR_PATH + "/data/sample-arabic-1.txt", + DIR_PATH + "/data/sample-french.txt", + DIR_PATH + "/data/sample-chinese.txt", + ] + ), + ) + + def test_with_alternative(self): + self.assertEqual( + 0, + cli_detect( + [ + "-a", + DIR_PATH + "/data/sample-arabic-1.txt", + DIR_PATH + "/data/sample-french.txt", + DIR_PATH + "/data/sample-chinese.txt", + ] + ), + ) + + def test_with_minimal_output(self): + self.assertEqual( + 0, + cli_detect( + [ + "-m", + DIR_PATH + "/data/sample-arabic-1.txt", + DIR_PATH + "/data/sample-french.txt", + DIR_PATH + "/data/sample-chinese.txt", + ] + ), + ) + + def test_with_minimal_and_alt(self): + self.assertEqual( + 0, + cli_detect( + [ + "-m", + "-a", + DIR_PATH + "/data/sample-arabic-1.txt", + DIR_PATH + "/data/sample-french.txt", + DIR_PATH + "/data/sample-chinese.txt", + ] + ), + ) + + def test_non_existent_file(self): + with self.assertRaises(SystemExit) as cm: + cli_detect([DIR_PATH + "/data/not_found_data.txt"]) + + self.assertEqual(cm.exception.code, 2) + + def test_replace_without_normalize(self): + self.assertEqual( + cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--replace"]), 1 + ) + + def test_force_replace_without_replace(self): + self.assertEqual( + cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--force"]), 1 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/retro-library/charset_normalizer/tests/test_coherence_detection.py b/tools/retro-library/charset_normalizer/tests/test_coherence_detection.py new file mode 100644 index 000000000..e5952d6c0 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_coherence_detection.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import pytest + +from charset_normalizer.cd import ( + encoding_languages, + filter_alt_coherence_matches, + get_target_features, + is_multi_byte_encoding, + mb_encoding_languages, +) + + +@pytest.mark.parametrize( + "iana_encoding, expected_languages", + [ + ("cp864", ["Arabic", "Farsi"]), + ("cp862", ["Hebrew"]), + ("cp737", ["Greek"]), + ("cp424", ["Hebrew"]), + ("cp273", ["Latin Based"]), + ("johab", ["Korean"]), + ("shift_jis", ["Japanese"]), + ("mac_greek", ["Greek"]), + ("iso2022_jp", ["Japanese"]), + ], +) +def test_infer_language_from_cp(iana_encoding, expected_languages): + languages = ( + mb_encoding_languages(iana_encoding) + if is_multi_byte_encoding(iana_encoding) + else encoding_languages(iana_encoding) + ) + + for expected_language in expected_languages: + assert ( + expected_language in languages + ), "Wrongly detected language for given code page" + + +@pytest.mark.parametrize( + "language, expected_have_accents, expected_pure_latin", + [ + ("English", False, True), + ("French", True, True), + ("Hebrew", False, False), + ("Arabic", False, False), + ("Vietnamese", True, True), + ("Turkish", True, True), + ], +) +def test_target_features(language, expected_have_accents, expected_pure_latin): + target_have_accents, target_pure_latin = get_target_features(language) + + assert target_have_accents is expected_have_accents + assert target_pure_latin is expected_pure_latin + + +@pytest.mark.parametrize( + "matches, expected_return", + [ + ( + [ + ( + "English", + 0.88, + ), + ("English—", 0.99), + ], + [("English", 0.99)], + ), + ( + [ + ( + "English", + 0.88, + ), + ("English—", 0.99), + ("English——", 0.999), + ], + [("English", 0.999)], + ), + ( + [ + ( + "English", + 0.88, + ), + ("English—", 0.77), + ], + [("English", 0.88)], + ), + ( + [ + ( + "English", + 0.88, + ), + ("Italian", 0.77), + ], + [("English", 0.88), ("Italian", 0.77)], + ), + ], +) +def test_filter_alt_coherence_matches(matches, expected_return): + results = filter_alt_coherence_matches(matches) + + assert results == expected_return diff --git a/tools/retro-library/charset_normalizer/tests/test_detect_legacy.py b/tools/retro-library/charset_normalizer/tests/test_detect_legacy.py new file mode 100644 index 000000000..bd2b03517 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_detect_legacy.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import unittest + +from charset_normalizer.legacy import detect + + +class TestDetectLegacy(unittest.TestCase): + def test_detect_dict_keys(self): + r = detect(("\uFEFF" + "我没有埋怨，磋砣的只是一些时间。").encode("gb18030")) + + with self.subTest("encoding key present"): + self.assertIn("encoding", r.keys()) + + with self.subTest("language key present"): + self.assertIn("language", r.keys()) + + with self.subTest("confidence key present"): + self.assertIn("confidence", r.keys()) + + def test_detect_dict_value_type(self): + r = detect("我没有埋怨，磋砣的只是一些时间。".encode()) + + with self.subTest("encoding instance of str"): + self.assertIsInstance(r["encoding"], str) + + with self.subTest("language instance of str"): + self.assertIsInstance(r["language"], str) + + with self.subTest("confidence instance of float"): + self.assertIsInstance(r["confidence"], float) + + def test_detect_dict_value(self): + r = detect("我没有埋怨，磋砣的只是一些时间。".encode("utf_32")) + + with self.subTest("encoding is equal to utf_32"): + self.assertEqual(r["encoding"], "UTF-32") + + def test_utf8_sig_not_striped(self): + r = detect("Hello World".encode("utf-8-sig")) + + with self.subTest("Verify that UTF-8-SIG is returned when using legacy detect"): + self.assertEqual(r["encoding"], "UTF-8-SIG") diff --git a/tools/retro-library/charset_normalizer/tests/test_edge_case.py b/tools/retro-library/charset_normalizer/tests/test_edge_case.py new file mode 100644 index 000000000..5b763ba27 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_edge_case.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import platform + +import pytest + +from charset_normalizer import from_bytes + + +@pytest.mark.xfail( + platform.python_version_tuple()[0] == "3" + and platform.python_version_tuple()[1] == "7", + reason="Unicode database is too old for this case (Python 3.7)", +) +def test_unicode_edge_case(): + payload = b"\xef\xbb\xbf\xf0\x9f\xa9\xb3" + + best_guess = from_bytes(payload).best() + + assert ( + best_guess is not None + ), "Payload should have given something, detection failure" + assert best_guess.encoding == "utf_8", "UTF-8 payload wrongly detected" + + +def test_issue_gh520(): + """Verify that minorities does not strip basic latin characters!""" + payload = b"/includes/webform.compon\xd2\xaants.inc/" + + best_guess = from_bytes(payload).best() + + assert ( + best_guess is not None + ), "Payload should have given something, detection failure" + assert "Basic Latin" in best_guess.alphabets + + +def test_issue_gh509(): + """Two common ASCII punctuations should render as-is.""" + payload = b");" + + best_guess = from_bytes(payload).best() + + assert ( + best_guess is not None + ), "Payload should have given something, detection failure" + assert "ascii" == best_guess.encoding + + +def test_issue_gh498(): + """This case was mistaken for utf-16-le, this should never happen again.""" + payload = b"\x84\xae\xaa\xe3\xac\xa5\xad\xe2 Microsoft Word.docx" + + best_guess = from_bytes(payload).best() + + assert ( + best_guess is not None + ), "Payload should have given something, detection failure" + assert "Cyrillic" in best_guess.alphabets diff --git a/tools/retro-library/charset_normalizer/tests/test_full_detection.py b/tools/retro-library/charset_normalizer/tests/test_full_detection.py new file mode 100644 index 000000000..ff91e1250 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_full_detection.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from os import pardir, path + +import pytest + +from charset_normalizer.api import from_path + +DIR_PATH = path.join(path.dirname(path.realpath(__file__)), pardir) + + +@pytest.mark.parametrize( + "input_data_file, expected_charset, expected_language", + [ + ("sample-arabic-1.txt", "cp1256", "Arabic"), + ("sample-french-1.txt", "cp1252", "French"), + ("sample-arabic.txt", "utf_8", "Arabic"), + ("sample-russian-3.txt", "utf_8", "Russian"), + ("sample-french.txt", "utf_8", "French"), + ("sample-chinese.txt", "big5", "Chinese"), + ("sample-greek.txt", "cp1253", "Greek"), + ("sample-greek-2.txt", "cp1253", "Greek"), + ("sample-hebrew-2.txt", "cp1255", "Hebrew"), + ("sample-hebrew-3.txt", "cp1255", "Hebrew"), + ("sample-bulgarian.txt", "utf_8", "Bulgarian"), + ("sample-english.bom.txt", "utf_8", "English"), + ("sample-spanish.txt", "utf_8", "Spanish"), + ("sample-korean.txt", "cp949", "Korean"), + ("sample-turkish.txt", "cp1254", "Turkish"), + ("sample-russian-2.txt", "utf_8", "Russian"), + ("sample-russian.txt", "mac_cyrillic", "Russian"), + ("sample-polish.txt", "utf_8", "Polish"), + ], +) +def test_elementary_detection( + input_data_file: str, + expected_charset: str, + expected_language: str, +): + best_guess = from_path(DIR_PATH + f"/data/{input_data_file}").best() + + assert ( + best_guess is not None + ), f"Elementary detection has failed upon '{input_data_file}'" + assert ( + best_guess.encoding == expected_charset + ), f"Elementary charset detection has failed upon '{input_data_file}'" + assert ( + best_guess.language == expected_language + ), f"Elementary language detection has failed upon '{input_data_file}'" diff --git a/tools/retro-library/charset_normalizer/tests/test_isbinary.py b/tools/retro-library/charset_normalizer/tests/test_isbinary.py new file mode 100644 index 000000000..841474f10 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_isbinary.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import typing +from base64 import b64decode +from io import BytesIO +from os import pardir, path + +import pytest + +from charset_normalizer import is_binary + +DIR_PATH = path.join(path.dirname(path.realpath(__file__)), pardir) + + +@pytest.mark.parametrize( + "raw, expected", + [ + (b"\x00\x5f\x2f\xff" * 50, True), + (b64decode("R0lGODlhAQABAAAAACw="), True), + (BytesIO(b64decode("R0lGODlhAQABAAAAACw=")), True), + ("sample-polish.txt", False), + ("sample-arabic.txt", False), + ], +) +def test_isbinary(raw: bytes | typing.BinaryIO | str, expected: bool) -> None: + if isinstance(raw, str): + raw = DIR_PATH + f"/data/{raw}" + + assert is_binary(raw) is expected diff --git a/tools/retro-library/charset_normalizer/tests/test_large_payload.py b/tools/retro-library/charset_normalizer/tests/test_large_payload.py new file mode 100644 index 000000000..7fc28fac1 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_large_payload.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import pytest + +from charset_normalizer import from_bytes +from charset_normalizer.constant import TOO_BIG_SEQUENCE + + +def test_large_payload_u8_sig_basic_entry(): + payload = ("0" * TOO_BIG_SEQUENCE).encode("utf_8_sig") + best_guess = from_bytes(payload).best() + + assert best_guess is not None, "Large U8 payload case detection completely failed" + assert ( + best_guess.encoding == "utf_8" + ), "Large U8 payload case detection wrongly detected!" + assert best_guess.bom is True, "SIG/BOM property should be True" + assert len(best_guess.raw) == len( + payload + ), "Large payload should remain untouched when accessed through .raw" + assert ( + best_guess._string is not None + ), "str should be decoded before direct access (sig available)" + + +def test_large_payload_ascii_basic_entry(): + payload = ("0" * TOO_BIG_SEQUENCE).encode("utf_8") + best_guess = from_bytes(payload).best() + + assert ( + best_guess is not None + ), "Large ASCII payload case detection completely failed" + assert ( + best_guess.encoding == "ascii" + ), "Large ASCII payload case detection wrongly detected!" + assert best_guess.bom is False, "SIG/BOM property should be False" + assert len(best_guess.raw) == len( + payload + ), "Large payload should remain untouched when accessed through .raw" + assert best_guess._string is None, "str should not be decoded until direct access" + + +def test_misleading_large_sequence(): + content = ( + ("hello simple ascii " * TOO_BIG_SEQUENCE) + ("我没有埋怨，磋砣的只是一些时间。 磋砣的只是一些时间。") + ).encode("utf_8") + + guesses = from_bytes(content) + + assert len(guesses) > 0 + match = guesses.best() + assert match is not None + assert match._string is not None, "str should be cached as only match" + assert match.encoding == "utf_8" + assert str(match) is not None diff --git a/tools/retro-library/charset_normalizer/tests/test_logging.py b/tools/retro-library/charset_normalizer/tests/test_logging.py new file mode 100644 index 000000000..ad2413e24 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_logging.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import logging + +import pytest + +from charset_normalizer.api import explain_handler, from_bytes +from charset_normalizer.constant import TRACE +from charset_normalizer.utils import set_logging_handler + + +class TestLogBehaviorClass: + def setup_method(self): + self.logger = logging.getLogger("charset_normalizer") + self.logger.handlers.clear() + self.logger.addHandler(logging.NullHandler()) + self.logger.level = logging.WARNING + + def test_explain_true_behavior(self, caplog): + test_sequence = b"This is a test sequence of bytes that should be sufficient" + from_bytes(test_sequence, steps=1, chunk_size=50, explain=True) + assert explain_handler not in self.logger.handlers + for record in caplog.records: + assert record.levelname in ["Level 5", "DEBUG"] + + def test_explain_false_handler_set_behavior(self, caplog): + test_sequence = b"This is a test sequence of bytes that should be sufficient" + set_logging_handler(level=TRACE, format_string="%(message)s") + from_bytes(test_sequence, steps=1, chunk_size=50, explain=False) + assert any( + isinstance(hdl, logging.StreamHandler) for hdl in self.logger.handlers + ) + for record in caplog.records: + assert record.levelname in ["Level 5", "DEBUG"] + assert "Encoding detection: ascii is most likely the one." in caplog.text + + def test_set_stream_handler(self, caplog): + set_logging_handler("charset_normalizer", level=logging.DEBUG) + self.logger.debug("log content should log with default format") + for record in caplog.records: + assert record.levelname in ["Level 5", "DEBUG"] + assert "log content should log with default format" in caplog.text + + def test_set_stream_handler_format(self, caplog): + set_logging_handler("charset_normalizer", format_string="%(message)s") + self.logger.info("log content should only be this message") + assert caplog.record_tuples == [ + ( + "charset_normalizer", + logging.INFO, + "log content should only be this message", + ) + ] diff --git a/tools/retro-library/charset_normalizer/tests/test_mess_detection.py b/tools/retro-library/charset_normalizer/tests/test_mess_detection.py new file mode 100644 index 000000000..4089f8259 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_mess_detection.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest + +from charset_normalizer.md import mess_ratio + + +@pytest.mark.parametrize( + "content, min_expected_ratio, max_expected_ratio", + [ + ( + "典肇乎庚辰年十二月廿一，及己丑年二月十九，收各方語言二百五十，合逾七百萬目；二十大卷佔八成，單英文卷亦過二百萬。悉文乃天下有志共筆而成；有意助之，幾網路、隨纂作，大典茁焉。", + 0.0, + 0.0, + ), + ("العقلية , التنويم المغناطيسي و / أو الاقتراح", 0.0, 0.0), + ("RadoZ تـــعــــديــل الـــتــــوقــيــــت مـــن قــبــل", 0.0, 0.0), + ("Cehennemin Sava■þ²s²'da kim?", 0.1, 0.5), + ("´Á¥½³ø§i -- ±i®Ìºû, ³¯·Ø©v", 0.5, 1.0), + ( + "ïstanbul, T■rkiye'nin en kalabal»k, iktisadi ve k■lt■rel aÓ»dan en —nemli", + 0.1, + 0.5, + ), + ( + "Parce que Óa, c'est la vÕritable histoire de la rencontre avec votre Tante Robin.", + 0.01, + 0.5, + ), + ( + """ØĢØŠØģاØĶŲ„ Ų„Ųˆ ØĢŲ† اŲ„Ų†Ø§Øģ ŲŠŲˆŲ… Ų…ا ØģŲˆŲŲŠØŠØģاØĶŲ„ŲˆŲ†ØŒ ØŊØđŲ†Ø§ Ų†ØģŲ…Øđ ØđŲ† (ŲØąŲˆØŊŲˆ) ŲˆØ§Ų„ØŪا؊Ų…""", + 0.8, + 3.0, + ), + ("""ÇáÚŞáíÉ , ÇáÊäæíã ÇáãÛäÇØíÓí æ / Ãæ ÇáÇŞÊÑÇÍ""", 0.8, 2.5), + ( + """hishamkoc@yahoo.com ุชุฑุฌู…ู€ู€ุฉ ู‡ู€ุดู€ู€ู€ุงู… ุงู„ู€ู‚ู€ู€ู€ู€ู„ุงูRadoZ ุชู€ู€ู€ุนู€ู€ู€ู€ุฏูŠู€ู€ู„ ุงู„ู€ู€ู€ุชู€ู€ู€ู€ูˆู‚ู€ู€ูŠู€ู€ู€ู€ุช ู…ู€ู€ู€ู† ู‚ู€ู€ุจู€ู€ู„""", + 0.5, + 2.0, + ), + ], +) +def test_mess_detection(content, min_expected_ratio, max_expected_ratio): + calculated_mess_ratio = mess_ratio(content, maximum_threshold=1.0) + + assert ( + min_expected_ratio <= calculated_mess_ratio <= max_expected_ratio + ), "The mess detection ratio calculated for given content is not well adjusted!" diff --git a/tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py b/tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py new file mode 100644 index 000000000..64b520235 --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import pytest + +from charset_normalizer import CharsetMatch +from charset_normalizer.utils import any_specified_encoding + + +@pytest.mark.parametrize( + "payload, expected_encoding", + [ + (b'', "euc_jp"), + (b'', "utf_8"), + (b'', None), + (b"# coding: utf-8", "utf_8"), + (b'', "utf_8"), + (b'', "ascii"), + (b'', "johab"), + (b'', "cp037"), + (b"", "cp1252"), + (b'', "cp1256"), + ], +) +def test_detect_most_common_body_encoding(payload, expected_encoding): + specified_encoding = any_specified_encoding(payload) + + assert ( + specified_encoding == expected_encoding + ), "Unable to determine properly encoding from given body" + + +@pytest.mark.parametrize( + "payload, expected_outcome", + [ + ( + b'', + b'', + ), + ( + b'', + b'', + ), + ( + b'', + b'', + ), + (b"# coding: utf-8", b"# coding: utf-8"), + ( + b'', + b'', + ), + ( + b'', + b'', + ), + ( + b'', + b'', + ), + ( + b"", + b"", + ), + ( + b'', + b'', + ), + ], +) +def test_preemptive_mark_replacement(payload, expected_outcome): + """ + When generating (to Unicode converted) bytes, we want to change any potential declarative charset + to utf-8. This test that. + """ + specified_encoding = any_specified_encoding(payload) + + detected_encoding = ( + specified_encoding if specified_encoding is not None else "utf-8" + ) + + m = CharsetMatch( + payload, + detected_encoding, + 0.0, + False, + [], + preemptive_declaration=specified_encoding, + ) + + transformed_output = m.output() + + assert transformed_output == expected_outcome diff --git a/tools/retro-library/charset_normalizer/tests/test_utils.py b/tools/retro-library/charset_normalizer/tests/test_utils.py new file mode 100644 index 000000000..a0cc088ed --- /dev/null +++ b/tools/retro-library/charset_normalizer/tests/test_utils.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import logging + +import pytest + +from charset_normalizer.utils import cp_similarity, is_accentuated, set_logging_handler + + +@pytest.mark.parametrize( + "character, expected_is_accentuated", + [ + ("é", True), + ("è", True), + ("à", True), + ("À", True), + ("Ù", True), + ("ç", True), + ("a", False), + ("€", False), + ("&", False), + ("Ö", True), + ("ü", True), + ("ê", True), + ("Ñ", True), + ("Ý", True), + ("Ω", False), + ("ø", False), + ("Ё", False), + ], +) +def test_is_accentuated(character, expected_is_accentuated): + assert ( + is_accentuated(character) is expected_is_accentuated + ), "is_accentuated behavior incomplete" + + +@pytest.mark.parametrize( + "cp_name_a, cp_name_b, expected_is_similar", + [ + ("cp1026", "cp1140", True), + ("cp1140", "cp1026", True), + ("latin_1", "cp1252", True), + ("latin_1", "iso8859_4", True), + ("latin_1", "cp1251", False), + ("cp1251", "mac_turkish", False), + ], +) +def test_cp_similarity(cp_name_a, cp_name_b, expected_is_similar): + is_similar = cp_similarity(cp_name_a, cp_name_b) >= 0.8 + + assert is_similar is expected_is_similar, "cp_similarity is broken" From 3edb9498a4c452892f1ccda4ddb035005db81802 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 19 Nov 2024 19:30:49 +0100 Subject: [PATCH 081/230] fix --- .../charset_normalizer/.codecov.yml | 8 - .../charset_normalizer/.coveragerc | 2 - .../charset_normalizer/.github/FUNDING.yml | 4 - .../.github/ISSUE_TEMPLATE/bug_report.md | 28 -- .../.github/ISSUE_TEMPLATE/feature_request.md | 20 - .../.github/ISSUE_TEMPLATE/wrong_charset.md | 39 -- .../charset_normalizer/.github/dependabot.yml | 20 - .../.github/workflows/cd.yml | 160 ------- .../.github/workflows/ci.yml | 252 ----------- .../.github/workflows/codeql.yml | 57 --- .../.github/workflows/scorecards.yml | 71 ---- .../charset_normalizer/.gitignore | 127 ------ .../.pre-commit-config.yaml | 31 -- .../charset_normalizer/.readthedocs.yaml | 19 - .../charset_normalizer/CHANGELOG.md | 393 ------------------ .../charset_normalizer/CODE_OF_CONDUCT.md | 76 ---- .../charset_normalizer/CONTRIBUTING.md | 72 ---- .../retro-library/charset_normalizer/LICENSE | 21 - .../charset_normalizer/MANIFEST.in | 5 - .../charset_normalizer/README.md | 257 ------------ .../charset_normalizer/SECURITY.md | 4 - .../charset_normalizer/UPGRADE.md | 31 -- .../{charset_normalizer => }/__init__.py | 0 .../{charset_normalizer => }/__main__.py | 0 .../{charset_normalizer => }/api.py | 0 .../charset_normalizer/bin/bc.py | 109 ----- .../charset_normalizer/bin/coverage.py | 106 ----- .../charset_normalizer/bin/integration.py | 56 --- .../charset_normalizer/bin/performance.py | 155 ------- .../charset_normalizer/bin/serve.py | 51 --- .../{charset_normalizer => }/cd.py | 0 .../{charset_normalizer => }/cli/__init__.py | 0 .../{charset_normalizer => }/cli/__main__.py | 0 .../{charset_normalizer => }/constant.py | 0 .../charset_normalizer/data/NOTICE.md | 9 - .../data/sample-arabic-1.txt | 6 - .../charset_normalizer/data/sample-arabic.txt | 6 - .../data/sample-bulgarian.txt | 7 - .../data/sample-chinese.txt | 14 - .../data/sample-english.bom.txt | 35 -- .../data/sample-french-1.txt | 59 --- .../charset_normalizer/data/sample-french.txt | 59 --- .../data/sample-greek-2.txt | 1 - .../charset_normalizer/data/sample-greek.txt | 1 - .../data/sample-hebrew-2.txt | 1 - .../data/sample-hebrew-3.txt | 1 - .../charset_normalizer/data/sample-korean.txt | 1 - .../charset_normalizer/data/sample-polish.txt | 204 --------- .../data/sample-russian-2.txt | 5 - .../data/sample-russian-3.txt | 7 - .../data/sample-russian.txt | 5 - .../data/sample-spanish.txt | 33 -- .../data/sample-turkish.txt | 33 -- .../charset_normalizer/dev-requirements.txt | 7 - .../charset_normalizer/docs/Makefile | 20 - .../charset_normalizer/docs/api.rst | 100 ----- .../charset_normalizer/docs/community/faq.rst | 75 ---- .../docs/community/featured.rst | 51 --- .../docs/community/speedup.rst | 50 --- .../docs/community/why_migrate.rst | 18 - .../charset_normalizer/docs/conf.py | 170 -------- .../charset_normalizer/docs/index.rst | 93 ----- .../charset_normalizer/docs/make.bat | 36 -- .../charset_normalizer/docs/requirements.txt | 2 - .../docs/user/advanced_search.rst | 82 ---- .../charset_normalizer/docs/user/cli.rst | 116 ------ .../docs/user/getstarted.rst | 73 ---- .../docs/user/handling_result.rst | 25 -- .../docs/user/miscellaneous.rst | 64 --- .../charset_normalizer/docs/user/support.rst | 183 -------- .../{charset_normalizer => }/legacy.py | 0 .../{charset_normalizer => }/md.py | 0 .../{charset_normalizer => }/models.py | 0 .../{charset_normalizer => }/py.typed | 0 .../charset_normalizer/pyproject.toml | 81 ---- .../charset_normalizer/setup.cfg | 3 - .../retro-library/charset_normalizer/setup.py | 30 -- .../charset_normalizer/tests/__init__.py | 0 .../tests/test_base_detection.py | 162 -------- .../charset_normalizer/tests/test_cli.py | 116 ------ .../tests/test_coherence_detection.py | 108 ----- .../tests/test_detect_legacy.py | 43 -- .../tests/test_edge_case.py | 59 --- .../tests/test_full_detection.py | 50 --- .../charset_normalizer/tests/test_isbinary.py | 29 -- .../tests/test_large_payload.py | 55 --- .../charset_normalizer/tests/test_logging.py | 53 --- .../tests/test_mess_detection.py | 48 --- .../tests/test_preemptive_detection.py | 92 ---- .../charset_normalizer/tests/test_utils.py | 52 --- .../{charset_normalizer => }/utils.py | 0 .../{charset_normalizer => }/version.py | 0 92 files changed, 4807 deletions(-) delete mode 100644 tools/retro-library/charset_normalizer/.codecov.yml delete mode 100644 tools/retro-library/charset_normalizer/.coveragerc delete mode 100644 tools/retro-library/charset_normalizer/.github/FUNDING.yml delete mode 100644 tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md delete mode 100644 tools/retro-library/charset_normalizer/.github/dependabot.yml delete mode 100644 tools/retro-library/charset_normalizer/.github/workflows/cd.yml delete mode 100644 tools/retro-library/charset_normalizer/.github/workflows/ci.yml delete mode 100644 tools/retro-library/charset_normalizer/.github/workflows/codeql.yml delete mode 100644 tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml delete mode 100644 tools/retro-library/charset_normalizer/.gitignore delete mode 100644 tools/retro-library/charset_normalizer/.pre-commit-config.yaml delete mode 100644 tools/retro-library/charset_normalizer/.readthedocs.yaml delete mode 100644 tools/retro-library/charset_normalizer/CHANGELOG.md delete mode 100644 tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md delete mode 100644 tools/retro-library/charset_normalizer/CONTRIBUTING.md delete mode 100644 tools/retro-library/charset_normalizer/LICENSE delete mode 100644 tools/retro-library/charset_normalizer/MANIFEST.in delete mode 100644 tools/retro-library/charset_normalizer/README.md delete mode 100644 tools/retro-library/charset_normalizer/SECURITY.md delete mode 100644 tools/retro-library/charset_normalizer/UPGRADE.md rename tools/retro-library/charset_normalizer/{charset_normalizer => }/__init__.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/__main__.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/api.py (100%) delete mode 100644 tools/retro-library/charset_normalizer/bin/bc.py delete mode 100644 tools/retro-library/charset_normalizer/bin/coverage.py delete mode 100644 tools/retro-library/charset_normalizer/bin/integration.py delete mode 100644 tools/retro-library/charset_normalizer/bin/performance.py delete mode 100644 tools/retro-library/charset_normalizer/bin/serve.py rename tools/retro-library/charset_normalizer/{charset_normalizer => }/cd.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/cli/__init__.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/cli/__main__.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/constant.py (100%) delete mode 100644 tools/retro-library/charset_normalizer/data/NOTICE.md delete mode 100644 tools/retro-library/charset_normalizer/data/sample-arabic-1.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-arabic.txt delete mode 100755 tools/retro-library/charset_normalizer/data/sample-bulgarian.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-chinese.txt delete mode 100755 tools/retro-library/charset_normalizer/data/sample-english.bom.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-french-1.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-french.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-greek-2.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-greek.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt delete mode 100755 tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-korean.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-polish.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-russian-2.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-russian-3.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-russian.txt delete mode 100755 tools/retro-library/charset_normalizer/data/sample-spanish.txt delete mode 100644 tools/retro-library/charset_normalizer/data/sample-turkish.txt delete mode 100644 tools/retro-library/charset_normalizer/dev-requirements.txt delete mode 100755 tools/retro-library/charset_normalizer/docs/Makefile delete mode 100644 tools/retro-library/charset_normalizer/docs/api.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/community/faq.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/community/featured.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/community/speedup.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/community/why_migrate.rst delete mode 100755 tools/retro-library/charset_normalizer/docs/conf.py delete mode 100755 tools/retro-library/charset_normalizer/docs/index.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/make.bat delete mode 100755 tools/retro-library/charset_normalizer/docs/requirements.txt delete mode 100644 tools/retro-library/charset_normalizer/docs/user/advanced_search.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/user/cli.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/user/getstarted.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/user/handling_result.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst delete mode 100644 tools/retro-library/charset_normalizer/docs/user/support.rst rename tools/retro-library/charset_normalizer/{charset_normalizer => }/legacy.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/md.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/models.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/py.typed (100%) delete mode 100644 tools/retro-library/charset_normalizer/pyproject.toml delete mode 100644 tools/retro-library/charset_normalizer/setup.cfg delete mode 100644 tools/retro-library/charset_normalizer/setup.py delete mode 100644 tools/retro-library/charset_normalizer/tests/__init__.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_base_detection.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_cli.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_coherence_detection.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_detect_legacy.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_edge_case.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_full_detection.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_isbinary.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_large_payload.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_logging.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_mess_detection.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py delete mode 100644 tools/retro-library/charset_normalizer/tests/test_utils.py rename tools/retro-library/charset_normalizer/{charset_normalizer => }/utils.py (100%) rename tools/retro-library/charset_normalizer/{charset_normalizer => }/version.py (100%) diff --git a/tools/retro-library/charset_normalizer/.codecov.yml b/tools/retro-library/charset_normalizer/.codecov.yml deleted file mode 100644 index f307895e7..000000000 --- a/tools/retro-library/charset_normalizer/.codecov.yml +++ /dev/null @@ -1,8 +0,0 @@ -coverage: - status: - project: - default: - target: 88% - threshold: null - patch: false - changes: false diff --git a/tools/retro-library/charset_normalizer/.coveragerc b/tools/retro-library/charset_normalizer/.coveragerc deleted file mode 100644 index 723bfd0cb..000000000 --- a/tools/retro-library/charset_normalizer/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -source=charset_normalizer diff --git a/tools/retro-library/charset_normalizer/.github/FUNDING.yml b/tools/retro-library/charset_normalizer/.github/FUNDING.yml deleted file mode 100644 index e793ecd50..000000000 --- a/tools/retro-library/charset_normalizer/.github/FUNDING.yml +++ /dev/null @@ -1,4 +0,0 @@ -# These are supported funding model platforms -tidelift: pypi/charset-normalizer -github: - - Ousret diff --git a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index a8be5653b..000000000 --- a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: Bug report -about: Create a report to help us fix something bad like an exception -title: "[BUG]" -labels: bug, help wanted -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug/exception is. - -**To Reproduce** -Give us the target text file. Host it somewhere with untouched encoding. - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Logs** -If applicable, add console outputs to help explain your problem. - -**Desktop (please complete the following information):** - - OS: [e.g. Linux, Windows or Mac] - - Python version [e.g. 3.5] - - Package version [eg. 2.0.0] - -**Additional context** -Add any other context about the problem here. diff --git a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 44d549f13..000000000 --- a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: "[Proposal]" -labels: enhancement -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md b/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md deleted file mode 100644 index e90d866d3..000000000 --- a/tools/retro-library/charset_normalizer/.github/ISSUE_TEMPLATE/wrong_charset.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: Wrong charset / Detection issue -about: Create a report to help us improve the detection mechanism -title: "[DETECTION]" -labels: help wanted, detection -assignees: '' - ---- - -**Notice** -I hereby announce that my raw input is not : -- Too small content (<=32 characters) as I do know that ANY charset detector heavily depends on content -- Encoded in a deprecated/abandoned encoding that is not even supported by my interpreter - -**Provide the file** -A accessible way of retrieving the file concerned. Host it somewhere with untouched encoding. - -**Verbose output** -Using the CLI, run `normalizer -v ./my-file.txt` and past the result in here. - -``` -(venv) >normalizer -v ./data/sample.1.ar.srt -2021-05-21 08:38:44,050 | DEBUG | ascii does not fit given bytes sequence at ALL. 'ascii' codec can't decode byte 0xca in position 54: ordinal not in range(128) -2021-05-21 08:38:44,051 | DEBUG | big5 does not fit given bytes sequence at ALL. 'big5' codec can't decode byte 0xc9 in position 60: illegal multibyte sequence -2021-05-21 08:38:44,051 | DEBUG | big5hkscs does not fit given bytes sequence at ALL. 'big5hkscs' codec can't decode byte 0xc9 in position 60: illegal multibyte sequence -.... -``` - -**Expected encoding** -A clear and concise description of what you expected as encoding. Any more details about how the current guess is wrong -is very much appreciated. - -**Desktop (please complete the following information):** - - OS: [e.g. Linux, Windows or Mac] - - Python version [e.g. 3.5] - - Package version [eg. 2.0.0] - -**Additional context** -Add any other context about the problem here. diff --git a/tools/retro-library/charset_normalizer/.github/dependabot.yml b/tools/retro-library/charset_normalizer/.github/dependabot.yml deleted file mode 100644 index d01328580..000000000 --- a/tools/retro-library/charset_normalizer/.github/dependabot.yml +++ /dev/null @@ -1,20 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - - - package-ecosystem: pip - directory: /docs - schedule: - interval: daily diff --git a/tools/retro-library/charset_normalizer/.github/workflows/cd.yml b/tools/retro-library/charset_normalizer/.github/workflows/cd.yml deleted file mode 100644 index 75ef10a66..000000000 --- a/tools/retro-library/charset_normalizer/.github/workflows/cd.yml +++ /dev/null @@ -1,160 +0,0 @@ -name: Continuous Delivery - -on: - workflow_dispatch: - - release: - types: - - created - -permissions: - contents: read - -jobs: - pre_flight_check: - name: Preflight Checks - uses: ./.github/workflows/ci.yml - - universal-wheel: - name: Build Universal Wheel - runs-on: ubuntu-latest - needs: - - pre_flight_check - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: '3.11' - - name: Update pip, install build - run: | - python -m pip install --upgrade pip - python -m pip install build - - name: Build Wheel - env: - CHARSET_NORMALIZER_USE_MYPYC: '0' - run: python -m build - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce - with: - name: dist - path: dist - - build-wheels: - name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }} - runs-on: ${{ matrix.os }} - needs: pre_flight_check - strategy: - matrix: - os: [ ubuntu-latest, windows-latest, macos-13 ] - qemu: [ '' ] - include: - # Split ubuntu job for the sake of speed-up - - os: ubuntu-latest - qemu: aarch64 - - os: ubuntu-latest - qemu: ppc64le - - os: ubuntu-latest - qemu: s390x - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: true - - name: Set up QEMU - if: ${{ matrix.qemu }} - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - with: - platforms: all - id: qemu - - name: Prepare emulation - run: | - if [[ -n "${{ matrix.qemu }}" ]]; then - # Build emulated architectures only if QEMU is set, - # use default "auto" otherwise - echo "CIBW_ARCHS_LINUX=${{ matrix.qemu }}" >> $GITHUB_ENV - fi - shell: bash - - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - - name: Update pip, wheel, setuptools, build, twine - run: | - python -m pip install -U pip wheel setuptools build twine - - name: Build wheels - uses: pypa/cibuildwheel@7940a4c0e76eb2030e473a5f864f291f63ee879b # v2.21.3 - env: - CIBW_BUILD_FRONTEND: build - CIBW_ARCHS_MACOS: x86_64 arm64 universal2 - CIBW_ENVIRONMENT: CHARSET_NORMALIZER_USE_MYPYC='1' - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: pytest -c {package} {package}/tests - CIBW_SKIP: pp* cp36* - - name: Upload artifacts - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce - with: - name: dist - path: ./wheelhouse/*.whl - - checksum: - name: Compute hashes - runs-on: ubuntu-latest - needs: - - build-wheels - - universal-wheel - outputs: - hashes: ${{ steps.compute.outputs.hashes }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Download distributions - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a - with: - name: dist - path: dist - - name: Collected dists - run: | - tree dist - - name: Generate hashes - id: compute # needs.checksum.outputs.hashes - working-directory: ./dist - run: echo "hashes=$(sha256sum * | base64 -w0)" >> $GITHUB_OUTPUT - - provenance: - needs: checksum - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 - permissions: - actions: read - id-token: write - contents: write - with: - base64-subjects: ${{ needs.checksum.outputs.hashes }} - upload-assets: true - compile-generator: true - - deploy: - name: 🚀 Deploy to PyPi - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - permissions: - id-token: write - contents: write - needs: provenance - environment: - name: pypi - url: https://pypi.org/project/charset-normalizer/ - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Download distributions - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a - with: - name: dist - path: dist - - name: Collected dists - run: | - tree dist - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@f7600683efdcb7656dec5b29656edb7bc586e597 # release/v1 - - name: Upload dists to GitHub Release - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - run: | - gh release upload ${{ github.ref_name }} dist/* --repo ${{ github.repository }} diff --git a/tools/retro-library/charset_normalizer/.github/workflows/ci.yml b/tools/retro-library/charset_normalizer/.github/workflows/ci.yml deleted file mode 100644 index 23433aab7..000000000 --- a/tools/retro-library/charset_normalizer/.github/workflows/ci.yml +++ /dev/null @@ -1,252 +0,0 @@ -name: Continuous Integration - -on: - workflow_call: - pull_request: - push: - branches: - - master - -permissions: - contents: read - -jobs: - lint: - name: 🎨 Linters - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install -U pip setuptools - python -m pip install -r dev-requirements.txt - python -m pip uninstall -y charset-normalizer - - name: Pre-commit checks - run: | - pre-commit run --all - - tests: - name: ✅ Tests - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: - - "3.7" - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Install dependencies - run: | - python -m pip install -U pip setuptools - python -m pip install -r dev-requirements.txt - python -m pip uninstall -y charset-normalizer - - name: Install the package - run: | - python -m build - python -m pip install ./dist/*.whl - - name: Run tests - run: | - pytest - - uses: codecov/codecov-action@4fe8c5f003fae66aa5ebb77cfd3e7bfbbda0b6b0 # v3.1.5 - - detection_coverage: - - needs: - - tests - - name: 📈 Detection Coverage - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install -U pip setuptools - python -m pip install -r dev-requirements.txt - python -m pip uninstall -y charset-normalizer - - name: Install the package - run: | - python -m build - python -m pip install ./dist/*.whl - - name: Clone the complete dataset - run: | - git clone https://github.com/Ousret/char-dataset.git - - name: Coverage WITH preemptive - run: | - python ./bin/coverage.py --coverage 97 --with-preemptive - - name: Coverage WITHOUT preemptive - run: | - python ./bin/coverage.py --coverage 95 - -# integration_test: -# -# needs: -# - tests -# -# name: 🔗 Integration Tests -# runs-on: ubuntu-latest -# -# steps: -# - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 -# - name: Set up Python -# uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 -# with: -# python-version: '3.11' -# - name: Install dependencies -# run: | -# pip install -U pip setuptools -# pip install -r dev-requirements.txt -# - name: Remove Chardet & Charset-Normalizer -# run: | -# pip uninstall -y chardet -# pip uninstall -y charset-normalizer -# - name: Install the package -# run: | -# python -m build -# pip install ./dist/*.whl -# - name: Clone the complete dataset -# run: | -# git clone https://github.com/Ousret/char-dataset.git -# - name: Start the Flask server -# run: | -# python ./bin/serve.py & -# - name: Integration Tests with Requests -# run: | -# python ./bin/integration.py - - chardet_bc: - - name: ⏪ Chardet Backward-Compatibility Test - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install -U pip setuptools - python -m pip install -r dev-requirements.txt - python -m pip uninstall -y charset-normalizer - - name: Install the package - run: | - python -m build - python -m pip install ./dist/*.whl - - name: Clone the complete dataset - run: | - git clone https://github.com/Ousret/char-dataset.git - - name: BC Coverage - run: | - python ./bin/bc.py --coverage 80 - - mypyc_test: - - name: ⚡ MypyC Tests - - needs: - - tests - - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - os: [ ubuntu-latest, macos-latest, windows-latest ] - include: - - python-version: "3.7" - os: ubuntu-latest - - python-version: "3.7" - os: macos-13 - - python-version: "3.7" - os: windows-latest - env: - PYTHONIOENCODING: utf8 # only needed for Windows (console IO output encoding) - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Install dependencies - run: | - python -m pip install -U pip setuptools - python -m pip install -r dev-requirements.txt - python -m pip uninstall -y charset-normalizer - - name: Install the package - env: - CHARSET_NORMALIZER_USE_MYPYC: '1' - run: | - python -m pip install . - - name: Clone the complete dataset - run: | - git clone https://github.com/Ousret/char-dataset.git - - name: Coverage WITH preemptive - run: | - python ./bin/coverage.py --coverage 97 --with-preemptive - - name: Performance (Normal) - run: | - python ./bin/performance.py - - performance: - name: ⚡ Performance Test (no MypyC) - runs-on: ubuntu-latest - - needs: - - mypyc_test - - chardet_bc - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: '3.11' - - name: Install dependencies - run: | - python -m pip install -U pip setuptools - python -m pip install -r dev-requirements.txt - python -m pip uninstall -y charset-normalizer - - name: Install the package - run: | - python -m build - python -m pip install ./dist/*.whl - - name: Clone the complete dataset - run: | - git clone https://github.com/Ousret/char-dataset.git - - name: Performance (Normal) - run: | - python ./bin/performance.py - - name: Performance (Medium) - run: | - python ./bin/performance.py --size-increase 2 diff --git a/tools/retro-library/charset_normalizer/.github/workflows/codeql.yml b/tools/retro-library/charset_normalizer/.github/workflows/codeql.yml deleted file mode 100644 index fee750870..000000000 --- a/tools/retro-library/charset_normalizer/.github/workflows/codeql.yml +++ /dev/null @@ -1,57 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -permissions: - contents: read - -on: - push: - branches: [ "master", "2.1.x" ] - pull_request: - branches: [ "master", "2.1.x" ] - schedule: - - cron: '39 1 * * 6' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - - steps: - - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 - with: - languages: ${{ matrix.language }} - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 - with: - category: "/language:${{matrix.language}}" diff --git a/tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml b/tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml deleted file mode 100644 index 2dbbaeff2..000000000 --- a/tools/retro-library/charset_normalizer/.github/workflows/scorecards.yml +++ /dev/null @@ -1,71 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: '20 7 * * 2' - push: - branches: ["master"] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - contents: read - actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecards on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1 - with: - sarif_file: results.sarif diff --git a/tools/retro-library/charset_normalizer/.gitignore b/tools/retro-library/charset_normalizer/.gitignore deleted file mode 100644 index d22c75f63..000000000 --- a/tools/retro-library/charset_normalizer/.gitignore +++ /dev/null @@ -1,127 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -.idea/ -char-dataset/ diff --git a/tools/retro-library/charset_normalizer/.pre-commit-config.yaml b/tools/retro-library/charset_normalizer/.pre-commit-config.yaml deleted file mode 100644 index 09fa625e5..000000000 --- a/tools/retro-library/charset_normalizer/.pre-commit-config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -exclude: 'docs/|data/|tests/' - -repos: - - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 - hooks: - - id: pyupgrade - args: ["--py37-plus"] - - - repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - args: ["--target-version", "py37"] - - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - additional_dependencies: [flake8-2020] - - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.13.0 - hooks: - - id: mypy - exclude: 'tests/|bin/' diff --git a/tools/retro-library/charset_normalizer/.readthedocs.yaml b/tools/retro-library/charset_normalizer/.readthedocs.yaml deleted file mode 100644 index b783a32fb..000000000 --- a/tools/retro-library/charset_normalizer/.readthedocs.yaml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2 - -build: - os: ubuntu-22.04 - tools: - python: "3.10" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: docs/conf.py - -# If using Sphinx, optionally build your docs in additional formats such as PDF -# formats: -# - pdf - -# Optionally declare the Python requirements required to build your docs -python: - install: - - requirements: docs/requirements.txt diff --git a/tools/retro-library/charset_normalizer/CHANGELOG.md b/tools/retro-library/charset_normalizer/CHANGELOG.md deleted file mode 100644 index 608567e41..000000000 --- a/tools/retro-library/charset_normalizer/CHANGELOG.md +++ /dev/null @@ -1,393 +0,0 @@ -# Changelog -All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - -## [3.4.1](https://github.com/Ousret/charset_normalizer/compare/3.4.0...master) (2024-10-??) - -### Changed -- Project metadata are now stored using `pyproject.toml` instead of `setup.cfg` using setuptools as the build backend. -- Enforce annotation delayed loading for a simpler and consistent types in the project. - -### Added -- pre-commit configuration. - -### Removed -- `build-requirements.txt` as per using `pyproject.toml` native build configuration. - -## [3.4.0](https://github.com/Ousret/charset_normalizer/compare/3.3.2...3.4.0) (2024-10-08) - -### Added -- Argument `--no-preemptive` in the CLI to prevent the detector to search for hints. -- Support for Python 3.13 (#512) - -### Fixed -- Relax the TypeError exception thrown when trying to compare a CharsetMatch with anything else than a CharsetMatch. -- Improved the general reliability of the detector based on user feedbacks. (#520) (#509) (#498) (#407) (#537) -- Declared charset in content (preemptive detection) not changed when converting to utf-8 bytes. (#381) - -## [3.3.2](https://github.com/Ousret/charset_normalizer/compare/3.3.1...3.3.2) (2023-10-31) - -### Fixed -- Unintentional memory usage regression when using large payload that match several encoding (#376) -- Regression on some detection case showcased in the documentation (#371) - -### Added -- Noise (md) probe that identify malformed arabic representation due to the presence of letters in isolated form (credit to my wife) - -## [3.3.1](https://github.com/Ousret/charset_normalizer/compare/3.3.0...3.3.1) (2023-10-22) - -### Changed -- Optional mypyc compilation upgraded to version 1.6.1 for Python >= 3.8 -- Improved the general detection reliability based on reports from the community - -## [3.3.0](https://github.com/Ousret/charset_normalizer/compare/3.2.0...3.3.0) (2023-09-30) - -### Added -- Allow to execute the CLI (e.g. normalizer) through `python -m charset_normalizer.cli` or `python -m charset_normalizer` -- Support for 9 forgotten encoding that are supported by Python but unlisted in `encoding.aliases` as they have no alias (#323) - -### Removed -- (internal) Redundant utils.is_ascii function and unused function is_private_use_only -- (internal) charset_normalizer.assets is moved inside charset_normalizer.constant - -### Changed -- (internal) Unicode code blocks in constants are updated using the latest v15.0.0 definition to improve detection -- Optional mypyc compilation upgraded to version 1.5.1 for Python >= 3.8 - -### Fixed -- Unable to properly sort CharsetMatch when both chaos/noise and coherence were close due to an unreachable condition in \_\_lt\_\_ (#350) - -## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07) - -### Changed -- Typehint for function `from_path` no longer enforce `PathLike` as its first argument -- Minor improvement over the global detection reliability - -### Added -- Introduce function `is_binary` that relies on main capabilities, and optimized to detect binaries -- Propagate `enable_fallback` argument throughout `from_bytes`, `from_path`, and `from_fp` that allow a deeper control over the detection (default True) -- Explicit support for Python 3.12 - -### Fixed -- Edge case detection failure where a file would contain 'very-long' camel cased word (Issue #289) - -## [3.1.0](https://github.com/Ousret/charset_normalizer/compare/3.0.1...3.1.0) (2023-03-06) - -### Added -- Argument `should_rename_legacy` for legacy function `detect` and disregard any new arguments without errors (PR #262) - -### Removed -- Support for Python 3.6 (PR #260) - -### Changed -- Optional speedup provided by mypy/c 1.0.1 - -## [3.0.1](https://github.com/Ousret/charset_normalizer/compare/3.0.0...3.0.1) (2022-11-18) - -### Fixed -- Multi-bytes cutter/chunk generator did not always cut correctly (PR #233) - -### Changed -- Speedup provided by mypy/c 0.990 on Python >= 3.7 - -## [3.0.0](https://github.com/Ousret/charset_normalizer/compare/2.1.1...3.0.0) (2022-10-20) - -### Added -- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results -- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES -- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio -- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl) - -### Changed -- Build with static metadata using 'build' frontend -- Make the language detection stricter -- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1 - -### Fixed -- CLI with opt --normalize fail when using full path for files -- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it -- Sphinx warnings when generating the documentation - -### Removed -- Coherence detector no longer return 'Simple English' instead return 'English' -- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese' -- Breaking: Method `first()` and `best()` from CharsetMatch -- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII) -- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches -- Breaking: Top-level function `normalize` -- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch -- Support for the backport `unicodedata2` - -## [3.0.0rc1](https://github.com/Ousret/charset_normalizer/compare/3.0.0b2...3.0.0rc1) (2022-10-18) - -### Added -- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results -- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES -- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio - -### Changed -- Build with static metadata using 'build' frontend -- Make the language detection stricter - -### Fixed -- CLI with opt --normalize fail when using full path for files -- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it - -### Removed -- Coherence detector no longer return 'Simple English' instead return 'English' -- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese' - -## [3.0.0b2](https://github.com/Ousret/charset_normalizer/compare/3.0.0b1...3.0.0b2) (2022-08-21) - -### Added -- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl) - -### Removed -- Breaking: Method `first()` and `best()` from CharsetMatch -- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII) - -### Fixed -- Sphinx warnings when generating the documentation - -## [3.0.0b1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...3.0.0b1) (2022-08-15) - -### Changed -- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1 - -### Removed -- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches -- Breaking: Top-level function `normalize` -- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch -- Support for the backport `unicodedata2` - -## [2.1.1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...2.1.1) (2022-08-19) - -### Deprecated -- Function `normalize` scheduled for removal in 3.0 - -### Changed -- Removed useless call to decode in fn is_unprintable (#206) - -### Fixed -- Third-party library (i18n xgettext) crashing not recognizing utf_8 (PEP 263) with underscore from [@aleksandernovikov](https://github.com/aleksandernovikov) (#204) - -## [2.1.0](https://github.com/Ousret/charset_normalizer/compare/2.0.12...2.1.0) (2022-06-19) - -### Added -- Output the Unicode table version when running the CLI with `--version` (PR #194) - -### Changed -- Re-use decoded buffer for single byte character sets from [@nijel](https://github.com/nijel) (PR #175) -- Fixing some performance bottlenecks from [@deedy5](https://github.com/deedy5) (PR #183) - -### Fixed -- Workaround potential bug in cpython with Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space (PR #175) -- CLI default threshold aligned with the API threshold from [@oleksandr-kuzmenko](https://github.com/oleksandr-kuzmenko) (PR #181) - -### Removed -- Support for Python 3.5 (PR #192) - -### Deprecated -- Use of backport unicodedata from `unicodedata2` as Python is quickly catching up, scheduled for removal in 3.0 (PR #194) - -## [2.0.12](https://github.com/Ousret/charset_normalizer/compare/2.0.11...2.0.12) (2022-02-12) - -### Fixed -- ASCII miss-detection on rare cases (PR #170) - -## [2.0.11](https://github.com/Ousret/charset_normalizer/compare/2.0.10...2.0.11) (2022-01-30) - -### Added -- Explicit support for Python 3.11 (PR #164) - -### Changed -- The logging behavior have been completely reviewed, now using only TRACE and DEBUG levels (PR #163 #165) - -## [2.0.10](https://github.com/Ousret/charset_normalizer/compare/2.0.9...2.0.10) (2022-01-04) - -### Fixed -- Fallback match entries might lead to UnicodeDecodeError for large bytes sequence (PR #154) - -### Changed -- Skipping the language-detection (CD) on ASCII (PR #155) - -## [2.0.9](https://github.com/Ousret/charset_normalizer/compare/2.0.8...2.0.9) (2021-12-03) - -### Changed -- Moderating the logging impact (since 2.0.8) for specific environments (PR #147) - -### Fixed -- Wrong logging level applied when setting kwarg `explain` to True (PR #146) - -## [2.0.8](https://github.com/Ousret/charset_normalizer/compare/2.0.7...2.0.8) (2021-11-24) -### Changed -- Improvement over Vietnamese detection (PR #126) -- MD improvement on trailing data and long foreign (non-pure latin) data (PR #124) -- Efficiency improvements in cd/alphabet_languages from [@adbar](https://github.com/adbar) (PR #122) -- call sum() without an intermediary list following PEP 289 recommendations from [@adbar](https://github.com/adbar) (PR #129) -- Code style as refactored by Sourcery-AI (PR #131) -- Minor adjustment on the MD around european words (PR #133) -- Remove and replace SRTs from assets / tests (PR #139) -- Initialize the library logger with a `NullHandler` by default from [@nmaynes](https://github.com/nmaynes) (PR #135) -- Setting kwarg `explain` to True will add provisionally (bounded to function lifespan) a specific stream handler (PR #135) - -### Fixed -- Fix large (misleading) sequence giving UnicodeDecodeError (PR #137) -- Avoid using too insignificant chunk (PR #137) - -### Added -- Add and expose function `set_logging_handler` to configure a specific StreamHandler from [@nmaynes](https://github.com/nmaynes) (PR #135) -- Add `CHANGELOG.md` entries, format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) (PR #141) - -## [2.0.7](https://github.com/Ousret/charset_normalizer/compare/2.0.6...2.0.7) (2021-10-11) -### Added -- Add support for Kazakh (Cyrillic) language detection (PR #109) - -### Changed -- Further, improve inferring the language from a given single-byte code page (PR #112) -- Vainly trying to leverage PEP263 when PEP3120 is not supported (PR #116) -- Refactoring for potential performance improvements in loops from [@adbar](https://github.com/adbar) (PR #113) -- Various detection improvement (MD+CD) (PR #117) - -### Removed -- Remove redundant logging entry about detected language(s) (PR #115) - -### Fixed -- Fix a minor inconsistency between Python 3.5 and other versions regarding language detection (PR #117 #102) - -## [2.0.6](https://github.com/Ousret/charset_normalizer/compare/2.0.5...2.0.6) (2021-09-18) -### Fixed -- Unforeseen regression with the loss of the backward-compatibility with some older minor of Python 3.5.x (PR #100) -- Fix CLI crash when using --minimal output in certain cases (PR #103) - -### Changed -- Minor improvement to the detection efficiency (less than 1%) (PR #106 #101) - -## [2.0.5](https://github.com/Ousret/charset_normalizer/compare/2.0.4...2.0.5) (2021-09-14) -### Changed -- The project now comply with: flake8, mypy, isort and black to ensure a better overall quality (PR #81) -- The BC-support with v1.x was improved, the old staticmethods are restored (PR #82) -- The Unicode detection is slightly improved (PR #93) -- Add syntax sugar \_\_bool\_\_ for results CharsetMatches list-container (PR #91) - -### Removed -- The project no longer raise warning on tiny content given for detection, will be simply logged as warning instead (PR #92) - -### Fixed -- In some rare case, the chunks extractor could cut in the middle of a multi-byte character and could mislead the mess detection (PR #95) -- Some rare 'space' characters could trip up the UnprintablePlugin/Mess detection (PR #96) -- The MANIFEST.in was not exhaustive (PR #78) - -## [2.0.4](https://github.com/Ousret/charset_normalizer/compare/2.0.3...2.0.4) (2021-07-30) -### Fixed -- The CLI no longer raise an unexpected exception when no encoding has been found (PR #70) -- Fix accessing the 'alphabets' property when the payload contains surrogate characters (PR #68) -- The logger could mislead (explain=True) on detected languages and the impact of one MBCS match (PR #72) -- Submatch factoring could be wrong in rare edge cases (PR #72) -- Multiple files given to the CLI were ignored when publishing results to STDOUT. (After the first path) (PR #72) -- Fix line endings from CRLF to LF for certain project files (PR #67) - -### Changed -- Adjust the MD to lower the sensitivity, thus improving the global detection reliability (PR #69 #76) -- Allow fallback on specified encoding if any (PR #71) - -## [2.0.3](https://github.com/Ousret/charset_normalizer/compare/2.0.2...2.0.3) (2021-07-16) -### Changed -- Part of the detection mechanism has been improved to be less sensitive, resulting in more accurate detection results. Especially ASCII. (PR #63) -- According to the community wishes, the detection will fall back on ASCII or UTF-8 in a last-resort case. (PR #64) - -## [2.0.2](https://github.com/Ousret/charset_normalizer/compare/2.0.1...2.0.2) (2021-07-15) -### Fixed -- Empty/Too small JSON payload miss-detection fixed. Report from [@tseaver](https://github.com/tseaver) (PR #59) - -### Changed -- Don't inject unicodedata2 into sys.modules from [@akx](https://github.com/akx) (PR #57) - -## [2.0.1](https://github.com/Ousret/charset_normalizer/compare/2.0.0...2.0.1) (2021-07-13) -### Fixed -- Make it work where there isn't a filesystem available, dropping assets frequencies.json. Report from [@sethmlarson](https://github.com/sethmlarson). (PR #55) -- Using explain=False permanently disable the verbose output in the current runtime (PR #47) -- One log entry (language target preemptive) was not show in logs when using explain=True (PR #47) -- Fix undesired exception (ValueError) on getitem of instance CharsetMatches (PR #52) - -### Changed -- Public function normalize default args values were not aligned with from_bytes (PR #53) - -### Added -- You may now use charset aliases in cp_isolation and cp_exclusion arguments (PR #47) - -## [2.0.0](https://github.com/Ousret/charset_normalizer/compare/1.4.1...2.0.0) (2021-07-02) -### Changed -- 4x to 5 times faster than the previous 1.4.0 release. At least 2x faster than Chardet. -- Accent has been made on UTF-8 detection, should perform rather instantaneous. -- The backward compatibility with Chardet has been greatly improved. The legacy detect function returns an identical charset name whenever possible. -- The detection mechanism has been slightly improved, now Turkish content is detected correctly (most of the time) -- The program has been rewritten to ease the readability and maintainability. (+Using static typing)+ -- utf_7 detection has been reinstated. - -### Removed -- This package no longer require anything when used with Python 3.5 (Dropped cached_property) -- Removed support for these languages: Catalan, Esperanto, Kazakh, Baque, Volapük, Azeri, Galician, Nynorsk, Macedonian, and Serbocroatian. -- The exception hook on UnicodeDecodeError has been removed. - -### Deprecated -- Methods coherence_non_latin, w_counter, chaos_secondary_pass of the class CharsetMatch are now deprecated and scheduled for removal in v3.0 - -### Fixed -- The CLI output used the relative path of the file(s). Should be absolute. - -## [1.4.1](https://github.com/Ousret/charset_normalizer/compare/1.4.0...1.4.1) (2021-05-28) -### Fixed -- Logger configuration/usage no longer conflict with others (PR #44) - -## [1.4.0](https://github.com/Ousret/charset_normalizer/compare/1.3.9...1.4.0) (2021-05-21) -### Removed -- Using standard logging instead of using the package loguru. -- Dropping nose test framework in favor of the maintained pytest. -- Choose to not use dragonmapper package to help with gibberish Chinese/CJK text. -- Require cached_property only for Python 3.5 due to constraint. Dropping for every other interpreter version. -- Stop support for UTF-7 that does not contain a SIG. -- Dropping PrettyTable, replaced with pure JSON output in CLI. - -### Fixed -- BOM marker in a CharsetNormalizerMatch instance could be False in rare cases even if obviously present. Due to the sub-match factoring process. -- Not searching properly for the BOM when trying utf32/16 parent codec. - -### Changed -- Improving the package final size by compressing frequencies.json. -- Huge improvement over the larges payload. - -### Added -- CLI now produces JSON consumable output. -- Return ASCII if given sequences fit. Given reasonable confidence. - -## [1.3.9](https://github.com/Ousret/charset_normalizer/compare/1.3.8...1.3.9) (2021-05-13) - -### Fixed -- In some very rare cases, you may end up getting encode/decode errors due to a bad bytes payload (PR #40) - -## [1.3.8](https://github.com/Ousret/charset_normalizer/compare/1.3.7...1.3.8) (2021-05-12) - -### Fixed -- Empty given payload for detection may cause an exception if trying to access the `alphabets` property. (PR #39) - -## [1.3.7](https://github.com/Ousret/charset_normalizer/compare/1.3.6...1.3.7) (2021-05-12) - -### Fixed -- The legacy detect function should return UTF-8-SIG if sig is present in the payload. (PR #38) - -## [1.3.6](https://github.com/Ousret/charset_normalizer/compare/1.3.5...1.3.6) (2021-02-09) - -### Changed -- Amend the previous release to allow prettytable 2.0 (PR #35) - -## [1.3.5](https://github.com/Ousret/charset_normalizer/compare/1.3.4...1.3.5) (2021-02-08) - -### Fixed -- Fix error while using the package with a python pre-release interpreter (PR #33) - -### Changed -- Dependencies refactoring, constraints revised. - -### Added -- Add python 3.9 and 3.10 to the supported interpreters diff --git a/tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md b/tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md deleted file mode 100644 index 0754df89c..000000000 --- a/tools/retro-library/charset_normalizer/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at ahmed.tahri@cloudnursery.dev. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq diff --git a/tools/retro-library/charset_normalizer/CONTRIBUTING.md b/tools/retro-library/charset_normalizer/CONTRIBUTING.md deleted file mode 100644 index abee674bd..000000000 --- a/tools/retro-library/charset_normalizer/CONTRIBUTING.md +++ /dev/null @@ -1,72 +0,0 @@ -# Contribution Guidelines - -If you’re reading this, you’re probably interested in contributing to Charset Normalizer. -Thank you very much! Open source projects live-and-die based on the support they receive from others, -and the fact that you’re even considering contributing to this project is very generous of you. - -## Questions - -The GitHub issue tracker is for *bug reports* and *feature requests*. -Questions are allowed only when no answer are provided in docs. - -## Good Bug Reports - -Please be aware of the following things when filing bug reports: - -1. Avoid raising duplicate issues. *Please* use the GitHub issue search feature - to check whether your bug report or feature request has been mentioned in - the past. Duplicate bug reports and feature requests are a huge maintenance - burden on the limited resources of the project. If it is clear from your - report that you would have struggled to find the original, that's ok, but - if searching for a selection of words in your issue title would have found - the duplicate then the issue will likely be closed extremely abruptly. -2. When filing bug reports about exceptions or tracebacks, please include the - *complete* traceback. Partial tracebacks, or just the exception text, are - not helpful. Issues that do not contain complete tracebacks may be closed - without warning. -3. Make sure you provide a suitable amount of information to work with. This - means you should provide: - - - Guidance on **how to reproduce the issue**. Ideally, this should be a - *small* code sample that can be run immediately by the maintainers. - Failing that, let us know what you're doing, how often it happens, what - environment you're using, etc. Be thorough: it prevents us needing to ask - further questions. - - Tell us **what you expected to happen**. When we run your example code, - what are we expecting to happen? What does "success" look like for your - code? - - Tell us **what actually happens**. It's not helpful for you to say "it - doesn't work" or "it fails". Tell us *how* it fails: do you get an - exception? A None answer? How was the actual result - different from your expected result? - - Tell us **what version of Charset Normalizer you're using**, and - **how you installed it**. Different versions of Charset Normalizer behave - differently and have different bugs. - - If you do not provide all of these things, it will take us much longer to - fix your problem. If we ask you to clarify these, and you never respond, we - will close your issue without fixing it. - - -## What PR are we accepting? - -Mostly anything, from cosmetic to the detection-mechanism improvement at the solo condition that you do not break -the backward-compatibility. - -## What PR may be doomed? - - - Add support for a Python unsupported charset/encoding -> If you looked carefully at the project, you would see that it aims to be generic whenever possible. So adding a specific prober is out of the question. - - - Of course, if the CI/CD are failing -> Getting the discussion started often mean doing the minimum effort to get it Green! (Be reassured, maintainers will look into it, given a reasonable amount of time) - - - Submitting a PR without any description OR viable commit description -> This is obvious, maintainers need to understand as fast as possible what are you trying to submit without putting too much effort. - -## How to run tests locally? - -It is essential that you run, prior to any submissions the mandatory checks. -Run the script `./bin/run_checks.sh` to verify that your modification are not breaking anything. - -Also, make sure to run the `./bin/run_autofix.sh` to comply with the style format and import sorting. diff --git a/tools/retro-library/charset_normalizer/LICENSE b/tools/retro-library/charset_normalizer/LICENSE deleted file mode 100644 index ad82355b8..000000000 --- a/tools/retro-library/charset_normalizer/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 TAHRI Ahmed R. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/MANIFEST.in b/tools/retro-library/charset_normalizer/MANIFEST.in deleted file mode 100644 index 3792f5bb9..000000000 --- a/tools/retro-library/charset_normalizer/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include LICENSE README.md CHANGELOG.md charset_normalizer/py.typed dev-requirements.txt -recursive-include data *.md -recursive-include data *.txt -recursive-include docs * -recursive-include tests * diff --git a/tools/retro-library/charset_normalizer/README.md b/tools/retro-library/charset_normalizer/README.md deleted file mode 100644 index 13e6e14ff..000000000 --- a/tools/retro-library/charset_normalizer/README.md +++ /dev/null @@ -1,257 +0,0 @@ -

Charset Detection, for Everyone 👋

- -

- The Real First Universal Charset Detector
- - - - - Download Count Total - - - - -

-

- Featured Packages
- - Static Badge - - - Static Badge - -

-

- In other language (unofficial port - by the community)
- - Static Badge - -

- -> A library that helps you read text from an unknown charset encoding.
Motivated by `chardet`, -> I'm trying to resolve the issue by taking a new approach. -> All IANA character set names for which the Python core library provides codecs are supported. - -

- >>>>> 👉 Try Me Online Now, Then Adopt Me 👈 <<<<< -

- -This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**. - -| Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) | -|--------------------------------------------------|:---------------------------------------------:|:--------------------------------------------------------------------------------------------------:|:-----------------------------------------------:| -| `Fast` | ❌ | ✅ | ✅ | -| `Universal**` | ❌ | ✅ | ❌ | -| `Reliable` **without** distinguishable standards | ❌ | ✅ | ✅ | -| `Reliable` **with** distinguishable standards | ✅ | ✅ | ✅ | -| `License` | LGPL-2.1
_restrictive_ | MIT | MPL-1.1
_restrictive_ | -| `Native Python` | ✅ | ✅ | ❌ | -| `Detect spoken language` | ❌ | ✅ | N/A | -| `UnicodeDecodeError Safety` | ❌ | ✅ | ❌ | -| `Whl Size (min)` | 193.6 kB | 42 kB | ~200 kB | -| `Supported Encoding` | 33 | 🎉 [99](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 | - -

-Reading Normalized TextCat Reading Text -

- -*\*\* : They are clearly using specific code for a specific encoding even if covering most of used one*
-Did you got there because of the logs? See [https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html](https://charset-normalizer.readthedocs.io/en/latest/user/miscellaneous.html) - -## ⚡ Performance - -This package offer better performance than its counterpart Chardet. Here are some numbers. - -| Package | Accuracy | Mean per file (ms) | File per sec (est) | -|-----------------------------------------------|:--------:|:------------------:|:------------------:| -| [chardet](https://github.com/chardet/chardet) | 86 % | 200 ms | 5 file/sec | -| charset-normalizer | **98 %** | **10 ms** | 100 file/sec | - -| Package | 99th percentile | 95th percentile | 50th percentile | -|-----------------------------------------------|:---------------:|:---------------:|:---------------:| -| [chardet](https://github.com/chardet/chardet) | 1200 ms | 287 ms | 23 ms | -| charset-normalizer | 100 ms | 50 ms | 5 ms | - -Chardet's performance on larger file (1MB+) are very poor. Expect huge difference on large payload. - -> Stats are generated using 400+ files using default parameters. More details on used files, see GHA workflows. -> And yes, these results might change at any time. The dataset can be updated to include more files. -> The actual delays heavily depends on your CPU capabilities. The factors should remain the same. -> Keep in mind that the stats are generous and that Chardet accuracy vs our is measured using Chardet initial capability -> (eg. Supported Encoding) Challenge-them if you want. - -## ✨ Installation - -Using pip: - -```sh -pip install charset-normalizer -U -``` - -## 🚀 Basic Usage - -### CLI -This package comes with a CLI. - -``` -usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD] - file [file ...] - -The Real First Universal Charset Detector. Discover originating encoding used -on text file. Normalize text to unicode. - -positional arguments: - files File(s) to be analysed - -optional arguments: - -h, --help show this help message and exit - -v, --verbose Display complementary information about file if any. - Stdout will contain logs about the detection process. - -a, --with-alternative - Output complementary possibilities if any. Top-level - JSON WILL be a list. - -n, --normalize Permit to normalize input file. If not set, program - does not write anything. - -m, --minimal Only output the charset detected to STDOUT. Disabling - JSON output. - -r, --replace Replace file when trying to normalize it instead of - creating a new one. - -f, --force Replace file without asking if you are sure, use this - flag with caution. - -t THRESHOLD, --threshold THRESHOLD - Define a custom maximum amount of chaos allowed in - decoded content. 0. <= chaos <= 1. - --version Show version information and exit. -``` - -```bash -normalizer ./data/sample.1.fr.srt -``` - -or - -```bash -python -m charset_normalizer ./data/sample.1.fr.srt -``` - -🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format. - -```json -{ - "path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt", - "encoding": "cp1252", - "encoding_aliases": [ - "1252", - "windows_1252" - ], - "alternative_encodings": [ - "cp1254", - "cp1256", - "cp1258", - "iso8859_14", - "iso8859_15", - "iso8859_16", - "iso8859_3", - "iso8859_9", - "latin_1", - "mbcs" - ], - "language": "French", - "alphabets": [ - "Basic Latin", - "Latin-1 Supplement" - ], - "has_sig_or_bom": false, - "chaos": 0.149, - "coherence": 97.152, - "unicode_path": null, - "is_preferred": true -} -``` - -### Python -*Just print out normalized text* -```python -from charset_normalizer import from_path - -results = from_path('./my_subtitle.srt') - -print(str(results.best())) -``` - -*Upgrade your code without effort* -```python -from charset_normalizer import detect -``` - -The above code will behave the same as **chardet**. We ensure that we offer the best (reasonable) BC result possible. - -See the docs for advanced usage : [readthedocs.io](https://charset-normalizer.readthedocs.io/en/latest/) - -## 😇 Why - -When I started using Chardet, I noticed that it was not suited to my expectations, and I wanted to propose a -reliable alternative using a completely different method. Also! I never back down on a good challenge! - -I **don't care** about the **originating charset** encoding, because **two different tables** can -produce **two identical rendered string.** -What I want is to get readable text, the best I can. - -In a way, **I'm brute forcing text decoding.** How cool is that ? 😎 - -Don't confuse package **ftfy** with charset-normalizer or chardet. ftfy goal is to repair unicode string whereas charset-normalizer to convert raw file in unknown encoding to unicode. - -## 🍰 How - - - Discard all charset encoding table that could not fit the binary content. - - Measure noise, or the mess once opened (by chunks) with a corresponding charset encoding. - - Extract matches with the lowest mess detected. - - Additionally, we measure coherence / probe for a language. - -**Wait a minute**, what is noise/mess and coherence according to **YOU ?** - -*Noise :* I opened hundred of text files, **written by humans**, with the wrong encoding table. **I observed**, then -**I established** some ground rules about **what is obvious** when **it seems like** a mess. - I know that my interpretation of what is noise is probably incomplete, feel free to contribute in order to - improve or rewrite it. - -*Coherence :* For each language there is on earth, we have computed ranked letter appearance occurrences (the best we can). So I thought -that intel is worth something here. So I use those records against decoded text to check if I can detect intelligent design. - -## ⚡ Known limitations - - - Language detection is unreliable when text contains two or more languages sharing identical letters. (eg. HTML (english tags) + Turkish content (Sharing Latin characters)) - - Every charset detector heavily depends on sufficient content. In common cases, do not bother run detection on very tiny content. - -## ⚠️ About Python EOLs - -**If you are running:** - -- Python >=2.7,<3.5: Unsupported -- Python 3.5: charset-normalizer < 2.1 -- Python 3.6: charset-normalizer < 3.1 -- Python 3.7: charset-normalizer < 4.0 - -Upgrade your Python interpreter as soon as possible. - -## 👤 Contributing - -Contributions, issues and feature requests are very much welcome.
-Feel free to check [issues page](https://github.com/ousret/charset_normalizer/issues) if you want to contribute. - -## 📝 License - -Copyright © [Ahmed TAHRI @Ousret](https://github.com/Ousret).
-This project is [MIT](https://github.com/Ousret/charset_normalizer/blob/master/LICENSE) licensed. - -Characters frequencies used in this project © 2012 [Denny Vrandečić](http://simia.net/letters/) - -## 💼 For Enterprise - -Professional support for charset-normalizer is available as part of the [Tidelift -Subscription][1]. Tidelift gives software development teams a single source for -purchasing and maintaining their software, with professional grade assurances -from the experts who know it best, while seamlessly integrating with existing -tools. - -[1]: https://tidelift.com/subscription/pkg/pypi-charset-normalizer?utm_source=pypi-charset-normalizer&utm_medium=readme diff --git a/tools/retro-library/charset_normalizer/SECURITY.md b/tools/retro-library/charset_normalizer/SECURITY.md deleted file mode 100644 index 005467cec..000000000 --- a/tools/retro-library/charset_normalizer/SECURITY.md +++ /dev/null @@ -1,4 +0,0 @@ -# Security Disclosures - -To report a security vulnerability, please use the [Tidelift security contact](https://tidelift.com/security). -Tidelift will coordinate the fix and disclosure with maintainers. diff --git a/tools/retro-library/charset_normalizer/UPGRADE.md b/tools/retro-library/charset_normalizer/UPGRADE.md deleted file mode 100644 index 4b8f7bb14..000000000 --- a/tools/retro-library/charset_normalizer/UPGRADE.md +++ /dev/null @@ -1,31 +0,0 @@ -Guide to upgrade your code from v1 to v2 ----------------------------------------- - - * If you are using the legacy `detect` function, that is it. You have nothing to do. - -## Detection - -### Before - -```python -from charset_normalizer import CharsetNormalizerMatches - -results = CharsetNormalizerMatches.from_bytes( - '我没有埋怨，磋砣的只是一些时间。'.encode('utf_32') -) -``` - -### After - -```python -from charset_normalizer import from_bytes - -results = from_bytes( - '我没有埋怨，磋砣的只是一些时间。'.encode('utf_32') -) -``` - -Methods that once were staticmethods of the class `CharsetNormalizerMatches` are now basic functions. -`from_fp`, `from_bytes`, `from_fp` and `` are concerned. - -Staticmethods scheduled to be removed in version 3.0 diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/__init__.py b/tools/retro-library/charset_normalizer/__init__.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/__init__.py rename to tools/retro-library/charset_normalizer/__init__.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/__main__.py b/tools/retro-library/charset_normalizer/__main__.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/__main__.py rename to tools/retro-library/charset_normalizer/__main__.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/api.py b/tools/retro-library/charset_normalizer/api.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/api.py rename to tools/retro-library/charset_normalizer/api.py diff --git a/tools/retro-library/charset_normalizer/bin/bc.py b/tools/retro-library/charset_normalizer/bin/bc.py deleted file mode 100644 index 4eacc1c49..000000000 --- a/tools/retro-library/charset_normalizer/bin/bc.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/python -from __future__ import annotations - -import argparse -from glob import glob -from os.path import isdir -from sys import argv - -from chardet import detect as chardet_detect - -from charset_normalizer import detect as tbt_detect -from charset_normalizer.utils import iana_name - - -def calc_equivalence(content: bytes, cp_a: str, cp_b: str): - try: - str_a = content.decode(cp_a) - str_b = content.decode(cp_b) - except UnicodeDecodeError: - return 0.0 - - character_count = len(str_a) - diff_character_count = sum(chr_a != chr_b for chr_a, chr_b in zip(str_a, str_b)) - - return 1.0 - (diff_character_count / character_count) - - -def cli_bc(arguments: list[str]): - parser = argparse.ArgumentParser( - description="BC script checker for Charset-Normalizer with Chardet" - ) - - parser.add_argument( - "-c", - "--coverage", - action="store", - default=85, - type=int, - dest="coverage", - help="Define the minimum acceptable coverage to succeed", - ) - - args = parser.parse_args(arguments) - - if not isdir("./char-dataset"): - print( - "This script require https://github.com/Ousret/char-dataset to be cloned on package root directory" - ) - exit(1) - - success_count = 0 - total_count = 0 - - for tbt_path in sorted(glob("./char-dataset/**/*.*")): - total_count += 1 - - with open(tbt_path, "rb") as fp: - content = fp.read() - - chardet_result = chardet_detect(content) - chardet_encoding = chardet_result["encoding"] - - charset_normalizer_result = tbt_detect(content) - charset_normalizer_encoding = charset_normalizer_result["encoding"] - - if [chardet_encoding, charset_normalizer_encoding].count(None) == 1: - print( - f"⚡⚡ '{tbt_path}' (BC-Break) New('{charset_normalizer_encoding}') vs Legacy('{chardet_encoding}')" - ) - continue - - if charset_normalizer_encoding == chardet_encoding: - success_count += 1 - print(f"✅✅ '{tbt_path}' (BC)") - continue - - if (chardet_encoding is None and charset_normalizer_encoding is None) or ( - iana_name(chardet_encoding, False) - == iana_name(charset_normalizer_encoding, False) - ): - success_count += 1 - print(f"✅✅ '{tbt_path}' (BC)") - continue - - calc_eq = calc_equivalence( - content, chardet_encoding, charset_normalizer_encoding - ) - - if calc_eq >= 0.98: - success_count += 1 - print( - f"️✅ ️'{tbt_path}' (got '{charset_normalizer_encoding}' but " - f"eq {chardet_encoding} WITH {round(calc_eq * 100., 3)} %)" - ) - continue - - print( - f"⚡⚡ '{tbt_path}' (BC-Break) New('{charset_normalizer_encoding}') vs Legacy('{chardet_encoding}')" - ) - - success_ratio = round(success_count / total_count, 2) * 100.0 - - print(f"Total EST BC = {success_ratio} % ({success_count} / {total_count} files)") - - return 0 if success_ratio >= args.coverage else 1 - - -if __name__ == "__main__": - exit(cli_bc(argv[1:])) diff --git a/tools/retro-library/charset_normalizer/bin/coverage.py b/tools/retro-library/charset_normalizer/bin/coverage.py deleted file mode 100644 index e5ba0110e..000000000 --- a/tools/retro-library/charset_normalizer/bin/coverage.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/python -from __future__ import annotations - -import argparse -from glob import glob -from os import sep -from os.path import isdir -from sys import argv - -from charset_normalizer import __version__, from_path -from charset_normalizer.utils import iana_name - - -def calc_equivalence(content: bytes, cp_a: str, cp_b: str): - str_a = content.decode(cp_a) - str_b = content.decode(cp_b) - - character_count = len(str_a) - diff_character_count = sum(chr_a != chr_b for chr_a, chr_b in zip(str_a, str_b)) - - return 1.0 - (diff_character_count / character_count) - - -def cli_coverage(arguments: list[str]): - parser = argparse.ArgumentParser( - description="Embedded detection success coverage script checker for Charset-Normalizer" - ) - - parser.add_argument( - "-p", - "--with-preemptive", - action="store_true", - default=False, - dest="preemptive", - help="Enable the preemptive scan behaviour during coverage check", - ) - parser.add_argument( - "-c", - "--coverage", - action="store", - default=90, - type=int, - dest="coverage", - help="Define the minimum acceptable coverage to succeed", - ) - - args = parser.parse_args(arguments) - - if not isdir("./char-dataset"): - print( - "This script require https://github.com/Ousret/char-dataset to be cloned on package root directory" - ) - exit(1) - - print(f"> using charset-normalizer {__version__}") - - success_count = 0 - total_count = 0 - - for tbt_path in sorted(glob("./char-dataset/**/*.*")): - expected_encoding = tbt_path.split(sep)[-2] - total_count += 1 - - results = from_path(tbt_path, preemptive_behaviour=args.preemptive) - - if expected_encoding == "None" and len(results) == 0: - print(f"✅✅ '{tbt_path}'") - success_count += 1 - continue - - if len(results) == 0: - print(f"⚡⚡ '{tbt_path}' (nothing)") - continue - - result = results.best() - - if ( - expected_encoding in result.could_be_from_charset - or iana_name(expected_encoding) in result.could_be_from_charset - ): - print(f"✅✅ '{tbt_path}'") - success_count += 1 - continue - - calc_eq = calc_equivalence(result.raw, expected_encoding, result.encoding) - - if calc_eq >= 0.98: - success_count += 1 - print( - f"️✅ ️'{tbt_path}' (got '{result.encoding}' but equivalence {round(calc_eq * 100., 3)} %)" - ) - continue - - print(f"⚡ '{tbt_path}' (got '{result.encoding}')") - - success_ratio = round(success_count / total_count, 2) * 100.0 - - print( - f"Total EST coverage = {success_ratio} % ({success_count} / {total_count} files)" - ) - - return 0 if success_ratio >= args.coverage else 1 - - -if __name__ == "__main__": - exit(cli_coverage(argv[1:])) diff --git a/tools/retro-library/charset_normalizer/bin/integration.py b/tools/retro-library/charset_normalizer/bin/integration.py deleted file mode 100644 index 7313ae5e0..000000000 --- a/tools/retro-library/charset_normalizer/bin/integration.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -from requests import __version__, get - -from charset_normalizer import __version__ as __version_cn__ -from charset_normalizer import detect - -if __name__ == "__main__": - print(f"requests {__version__}") - print(f"charset_normalizer {__version_cn__}") - - files: list[str] = get("http://127.0.0.1:8080/").json() - - print("## Testing with actual files") - - for file in files: - r = get("http://127.0.0.1:8080/" + file) - - if r.ok is False: - print(f"Unable to retrieve '{file}' | HTTP/{r.status_code}") - exit(1) - - expected_encoding = detect(r.content)["encoding"] - - if expected_encoding != r.apparent_encoding: - print( - f"Integration test failed | File '{file}' | Expected '{expected_encoding}' got '{r.apparent_encoding}'" - ) - exit(1) - - print(f"✅✅ '{file}' OK") - - print("## Testing with edge cases") - - # Should NOT crash - get("http://127.0.0.1:8080/edge/empty/json").json() - - print("✅✅ Empty JSON OK") - - if get("http://127.0.0.1:8080/edge/empty/plain").apparent_encoding != "utf-8": - print("Empty payload SHOULD not return apparent_encoding != UTF-8") - exit(1) - - print("✅✅ Empty Plain Text OK") - - r = get("http://127.0.0.1:8080/edge/gb18030/json") - - if r.apparent_encoding != "GB18030": - print("JSON Basic Detection FAILURE (/edge/gb18030/json)") - exit(1) - - r.json() - - print("✅✅ GB18030 JSON Encoded OK") - - print("Integration tests passed!") diff --git a/tools/retro-library/charset_normalizer/bin/performance.py b/tools/retro-library/charset_normalizer/bin/performance.py deleted file mode 100644 index 41195b8f8..000000000 --- a/tools/retro-library/charset_normalizer/bin/performance.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/python -from __future__ import annotations - -import argparse -from glob import glob -from math import ceil -from os.path import isdir -from statistics import mean, stdev -from sys import argv -from time import perf_counter_ns - -from chardet import detect as chardet_detect - -from charset_normalizer import detect - - -def calc_percentile(data, percentile): - n = len(data) - p = n * percentile / 100 - sorted_data = sorted(data) - - return sorted_data[int(p)] if p.is_integer() else sorted_data[int(ceil(p)) - 1] - - -def performance_compare(arguments): - parser = argparse.ArgumentParser( - description="Performance CI/CD check for Charset-Normalizer" - ) - - parser.add_argument( - "-s", - "--size-increase", - action="store", - default=1, - type=int, - dest="size_coeff", - help="Apply artificial size increase to challenge the detection mechanism further", - ) - - args = parser.parse_args(arguments) - - if not isdir("./char-dataset"): - print( - "This script require https://github.com/Ousret/char-dataset to be cloned on package root directory" - ) - exit(1) - - chardet_results = [] - charset_normalizer_results = [] - - file_list = sorted(glob("./char-dataset/**/*.*")) - total_files = len(file_list) - - for idx, tbt_path in enumerate(file_list): - with open(tbt_path, "rb") as fp: - content = fp.read() * args.size_coeff - - before = perf_counter_ns() - chardet_detect(content) - chardet_time = round((perf_counter_ns() - before) / 1000000000, 5) - chardet_results.append(chardet_time) - - before = perf_counter_ns() - detect(content) - charset_normalizer_time = round((perf_counter_ns() - before) / 1000000000, 5) - charset_normalizer_results.append(charset_normalizer_time) - - charset_normalizer_time = charset_normalizer_time or 0.000005 - cn_faster = (chardet_time / charset_normalizer_time) * 100 - 100 - print( - f"{idx + 1:>3}/{total_files} {tbt_path:<82} C:{chardet_time:.5f} " - f"CN:{charset_normalizer_time:.5f} {cn_faster:.1f} %" - ) - - # Print the top 10 rows with the slowest execution time - print( - f"\n{'-' * 102}\nTop 10 rows with the slowest execution time of charset_normalizer:\n" - ) - sorted_results = sorted( - enumerate(charset_normalizer_results), key=lambda x: x[1], reverse=True - ) - for idx, time in sorted_results[:10]: - tbt_path = file_list[idx] - print(f"{idx + 1:>3}/{total_files} {tbt_path:<82} CN:{time:.5f}") - - # Print charset normalizer statistics - min_time = min(charset_normalizer_results) - max_time = max(charset_normalizer_results) - stdev_time = stdev(charset_normalizer_results) - mean_time = mean(charset_normalizer_results) - cv = (stdev_time / mean_time) * 100 # Coefficient of variation - print(f"\n{'-' * 102}\nCharset Normalizer statistics:\n") - print(f"Minimum Execution Time: {min_time:.5f} seconds") - print(f"Maximum Execution Time: {max_time:.5f} seconds") - print(f"Mean Execution Time: {mean_time:.5f} seconds") - print(f"Standard Deviation: {stdev_time:.5f} seconds") - print(f"Coefficient of Variation (CV): {cv:.1f} %") - - # Print comparison statistics for chardet and charset normalizer - chardet_avg_delay = round(mean(chardet_results) * 1000) - chardet_99p = round(calc_percentile(chardet_results, 99) * 1000) - chardet_95p = round(calc_percentile(chardet_results, 95) * 1000) - chardet_50p = round(calc_percentile(chardet_results, 50) * 1000) - - charset_normalizer_avg_delay = round(mean(charset_normalizer_results) * 1000) - charset_normalizer_99p = round( - calc_percentile(charset_normalizer_results, 99) * 1000 - ) - charset_normalizer_95p = round( - calc_percentile(charset_normalizer_results, 95) * 1000 - ) - charset_normalizer_50p = round( - calc_percentile(charset_normalizer_results, 50) * 1000 - ) - - # mypyc can offer performance ~1ms in the 50p. When eq to 0 assume 1 due to imprecise nature of this test. - if charset_normalizer_50p == 0: - charset_normalizer_50p = 1 - - print(f"\n{'-' * 102}\nCharset Normalizer vs Chardet statistics:\n") - - print("------------------------------") - print("--> Chardet Conclusions") - print(" --> Avg: " + str(chardet_avg_delay) + "ms") - print(" --> 99th: " + str(chardet_99p) + "ms") - print(" --> 95th: " + str(chardet_95p) + "ms") - print(" --> 50th: " + str(chardet_50p) + "ms") - - print("------------------------------") - print("--> Charset-Normalizer Conclusions") - print(" --> Avg: " + str(charset_normalizer_avg_delay) + "ms") - print(" --> 99th: " + str(charset_normalizer_99p) + "ms") - print(" --> 95th: " + str(charset_normalizer_95p) + "ms") - print(" --> 50th: " + str(charset_normalizer_50p) + "ms") - - print("------------------------------") - print("--> Charset-Normalizer / Chardet: Performance Сomparison") - print( - " --> Avg: x" - + str(round(chardet_avg_delay / charset_normalizer_avg_delay, 2)) - ) - print(" --> 99th: x" + str(round(chardet_99p / charset_normalizer_99p, 2))) - print(" --> 95th: x" + str(round(chardet_95p / charset_normalizer_95p, 2))) - print(" --> 50th: x" + str(round(chardet_50p / charset_normalizer_50p, 2))) - - return ( - 0 - if chardet_avg_delay > charset_normalizer_avg_delay - and chardet_99p > charset_normalizer_99p - else 1 - ) - - -if __name__ == "__main__": - exit(performance_compare(argv[1:])) diff --git a/tools/retro-library/charset_normalizer/bin/serve.py b/tools/retro-library/charset_normalizer/bin/serve.py deleted file mode 100644 index 99d6b6ea4..000000000 --- a/tools/retro-library/charset_normalizer/bin/serve.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -from glob import glob - -from flask import Flask, jsonify, send_from_directory - -app = Flask(__name__) - - -@app.route("/raw/") -def read_file(path): - return ( - send_from_directory("../char-dataset", path, as_attachment=True), - 200, - {"Content-Type": "text/plain"}, - ) - - -@app.route("/") -def read_targets(): - return jsonify( - [ - el.replace("./char-dataset", "/raw").replace("\\", "/") - for el in sorted(glob("./char-dataset/**/*")) - ] - ) - - -@app.route("/edge/empty/plain") -def read_empty_response_plain(): - return b"", 200, {"Content-Type": "text/plain"} - - -@app.route("/edge/empty/json") -def read_empty_response_json(): - return b"{}", 200, {"Content-Type": "application/json"} - - -@app.route("/edge/gb18030/json") -def read_gb18030_response_json(): - return ( - '{"abc": "我没有埋怨，磋砣的只是一些时间。。今觀俗士之論也，以族舉德，以位命賢，茲可謂得論之一體矣，而未獲至論之淑真也。"}'.encode( - "gb18030" - ), - 200, - {"Content-Type": "application/json"}, - ) - - -if __name__ == "__main__": - app.run(host="127.0.0.1", port=8080) diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/cd.py b/tools/retro-library/charset_normalizer/cd.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/cd.py rename to tools/retro-library/charset_normalizer/cd.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/cli/__init__.py b/tools/retro-library/charset_normalizer/cli/__init__.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/cli/__init__.py rename to tools/retro-library/charset_normalizer/cli/__init__.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/cli/__main__.py b/tools/retro-library/charset_normalizer/cli/__main__.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/cli/__main__.py rename to tools/retro-library/charset_normalizer/cli/__main__.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/constant.py b/tools/retro-library/charset_normalizer/constant.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/constant.py rename to tools/retro-library/charset_normalizer/constant.py diff --git a/tools/retro-library/charset_normalizer/data/NOTICE.md b/tools/retro-library/charset_normalizer/data/NOTICE.md deleted file mode 100644 index 4154617ca..000000000 --- a/tools/retro-library/charset_normalizer/data/NOTICE.md +++ /dev/null @@ -1,9 +0,0 @@ -Included and Redistributed Files ---------------------------------- - -17 files are included in the source distribution tar. They are used to verify the standard functions of -this library. They are mandatory to run `pytest` but not required to make the lib usable after install. -They DO NOT guarantee that the detection-coverage will not regress. - -Those are EITHER pulled from Wikipedia _(CC-BY-SA)_ OR public domain archive. -You SHALL NOT modify any of those files without explicit approval. diff --git a/tools/retro-library/charset_normalizer/data/sample-arabic-1.txt b/tools/retro-library/charset_normalizer/data/sample-arabic-1.txt deleted file mode 100644 index 5648760e2..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-arabic-1.txt +++ /dev/null @@ -1,6 +0,0 @@ - " " " " : ( ) ( ) ( ). - - : " " " " " " ߡ . - - ( : ) ( : ). ( : Morroch) - diff --git a/tools/retro-library/charset_normalizer/data/sample-arabic.txt b/tools/retro-library/charset_normalizer/data/sample-arabic.txt deleted file mode 100644 index 4c18892b8..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-arabic.txt +++ /dev/null @@ -1,6 +0,0 @@ -بالموازاة مع ذلك وللإشارة إلى المنطقة المغاربية بشكل عام، كان المؤرخون العرب في القرون الوسطى يستعملون لفظ "بلاد المغرب" بينما الأوروبيون يستعملون لفظ "الساحل البربري" للدلالة على ثلاثة أقاليم: المغرب الأدنى (إفريقية أو تونس الحالية)، المغرب الأوسط (الجزائر الحالية)، المغرب الأقصى (المملكة المغربية الحالية). - -أحيانًا كان يُشار للبلاد بتسمية مرتبطة بعاصمتها: كـ "موريطنية الطنجية" التي كانت عاصمتها طنجة وكذا "مملكة مراكش" و"مملكة فاس" نسبة إلى عواصمها المعروفة آنذاك، وكانت الظهائر والمعاهدات الدولية يوقّعها سلاطين المغرب تارة باسم سلطان مراكش وتارة باسم سلطان فاس. - -تمت الإشارة للبلاد لاحقًا باسم المغرب الأقصى باللغة العربية حيث اعتَقد الناس في العالم القديم أن الشمس تشرق من اليابان (باللغة الصينية نيهون: مكان شروق الشمس) وتغرب في المملكة المغربية (باللغة العربية المغرب: مكان غروب الشمس). بينما اشتَقت البلاد اسمها في اللغات الأوروبية من الكلمة اللاتينية مرك (باللغة اللاتينية: Morroch) وهي تصحيف - diff --git a/tools/retro-library/charset_normalizer/data/sample-bulgarian.txt b/tools/retro-library/charset_normalizer/data/sample-bulgarian.txt deleted file mode 100755 index 7a3ad68a3..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-bulgarian.txt +++ /dev/null @@ -1,7 +0,0 @@ -Член 26 -1. Bсеки човек има право на образование. Oбразованието трябва да бъде безплатно, поне що се отнася до началното и основното образование. Hачалното образование трябва да бъде задължително. Tехническото и професионалното образование трябва да бъдат общодостъпни, а висшето образование трябва да бъде еднакво достъпно за всички на основата на техните способности. -2. Oбразованието трябва да бъде насочено към цялостно развитие на човешката личност и заcилване на уважението към правата на човека и основните свободи. Tо трябва да съдейства за разбирателството, тъпримостта и приятелството между всички народи, расови или религиозни групи, както и за осъществяване дейността на Oрганизацията на Oбединените нации за поддържане на мира. -3. Pодителите имат право, с приоритет, да избират вида образование, което да получат техните деца. -Член 27 -1. Bсеки човек има право свободно да участва в културния живот на обществото, да се наслаждава на изкуствата, да участва в научния напредък и да се ползва от неговите достижения. -2. Bсеки човек има право на закрила на моралните и материалните си интереси, които са резултат от каквото и да е научно, литературно или художествено произведение, на което той е автор. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/data/sample-chinese.txt b/tools/retro-library/charset_normalizer/data/sample-chinese.txt deleted file mode 100644 index 760df5af8..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-chinese.txt +++ /dev/null @@ -1,14 +0,0 @@ -j]Wikipedia^̡A¦FѤUB|BHӡAѦʬjCl@̡AC|]C - -FG~QGܤ@AΤv~GQEAUyGʤQAXOCʸUءFGQjKA^LGʸUCxDѤUӦ@ӦFNUAXBHġ@Aj_jC - -@AXjոܺCmlnJuAô]vAmnJuAl]vAHXANôl]C´AHڥA]C - -joAPDݵoFpsDBХBBӾǡByBwCئhiw\שsF\׽֡AѱoPġA_ɨaHAMhҸչAKGC - -Z򤧵AҾڪ̡AꭲۥѤɳ\iijAGiۥѼsǤѤUաC - -娥l~CiAoGd@ʤKQEC - -commons: - nHMTA@ɡJjC diff --git a/tools/retro-library/charset_normalizer/data/sample-english.bom.txt b/tools/retro-library/charset_normalizer/data/sample-english.bom.txt deleted file mode 100755 index 7c87e9489..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-english.bom.txt +++ /dev/null @@ -1,35 +0,0 @@ -1 -00:00:06,500 --> 00:00:09,000 -About 2 months ago I found myself on -the comment section of YouTube - -2 -00:00:11,000 --> 00:00:17,000 -And I was commenting, -unfortunately I was commenting, -on a video about the famous Ayn Rand - -3 -00:00:19,000 --> 00:00:24,000 -And I -posted underneath against -this woman's tirades, -against what is essentially -the human race. - -4 -00:00:25,000 --> 00:00:31,000 -that, this monetary system seems to have no point, seems to actually hinder people - -5 -00:00:31,000 --> 00:00:36,000 -and hinder progress, and one of the responses I got, I didn't answer it, was: - -6 -00:00:37,000 --> 00:00:43,000 -what actually money creates is an incentive to invent the new items, that's the driving force behind it - -7 -00:00:43,000 --> 00:00:50,000 -So what I thought I do is instead if answering on a YouTube comment is organize a global awareness day - diff --git a/tools/retro-library/charset_normalizer/data/sample-french-1.txt b/tools/retro-library/charset_normalizer/data/sample-french-1.txt deleted file mode 100644 index f87bdf715..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-french-1.txt +++ /dev/null @@ -1,59 +0,0 @@ -JEAN-BAPTISTE POQUELIN MOLIRE -N LE 15 JANVIER 1622, MORT LE 17 FVRIER 1673 - - -Quel est le plus grand des crivains de mon rgne? demandait Louis XIV - Boileau.--Sire, c'est Molire. - -Non-seulement Despraux ne se trompait pas, mais de tous les crivains -que la France a produits, sans excepter Voltaire lui-mme, imprgn de -l'esprit anglais par son sjour Londres, c'est incontestablement -Molire ou Poquelin qui reproduit avec l'exactitude la plus vive et la -plus complte le fond du gnie franais. - -En raison de cette identit de son gnie avec le ntre, il exera sur -l'poque subsquente, sur le dix-huitime sicle, sur l'poque mme o -nous crivons, la plus active, la plus redoutable influence. Tout ce -qu'il a voulu dtruire est en ruine. Les types qu'il a crs ne peuvent -mourir. Le sens de la vie pratique, qu'il a recommand d'aprs Gassendi, -a fini par l'emporter sur les ides qui imposaient la socit -franaise. Il n'y a pas de superstition qu'il n'ait attaque, pas de -crdulit qu'il n'ait saisie corps corps pour la terrasser, pas de -formule qu'il ne se soit efforc de dtruire. A-t-il, comme l'exprime si -bien Swift, _dchir l'toffe avec la doublure_? l'histoire le dira. Ce -qui est certain, c'est que l'lve de Lucrce, le protg de Louis XIV, -poursuivait un but dtermin vers lequel il a march d'un pas ferme, -obstin, tantt foulant aux pieds les obstacles, tantt les tournant -avec adresse. Le sujet de _Tartuffe_ est dans Lucrce; Lucrce -appartient ce vers, vritable devise de Molire: - - _Et religionis..... nodos solvere curo[1]._ - -La puissance de Molire sur les esprits a t telle, qu'une lgende -inexacte, calomnieuse de son vivant, romanesque aprs sa mort, s'est -forme autour de cette gloire populaire. Il est un mythe comme Jules -Csar et Apollon. - - [1] Ce que je veux, c'est rompre les entraves qui nous enchanent - (_religionis.... quod religat_). - -Dates, vnements, ralits, souvenirs, sont venus se confondre dans un -inextricable chaos o la figure de Molire a disparu. Tous les vices -jusqu' l'ivrognerie, jusqu' l'inceste et au vol, lui furent imputs de -son vivant. Les vertus les plus thres lui furent attribues par les -prtres de son culte. Homme d'action, sans cesse en face du public, du -roi ou de sa troupe, occup de son gouvernement et de la cration de ses -uvres, il n'a laiss aucune trace de sa propre vie, aucun document -biographique, peine une lettre. Les pamphlets pour et contre lui -composaient dj une bibliothque, lorsqu'un couteur aux portes, nomm -Grimarest, collecteur d'anas, aimant l'exagration des rcits et -incapable de critique, prtendit, trente-deux ans aprs la mort du -comdien populaire, raconter et expliquer sa vie. Vers la mme poque, -une comdienne, ce que l'on croit du moins, force de se rfugier en -Hollande, jetait dans un libelle les souvenirs de coulisse qu'elle avait -pu recueillir sur l'intrieur du mnage de Molire et de sa femme. Enfin -quelques dtails authentiques, sems dans l'dition de ses uvres -publie par Lagrange en 1682, compltent l'ensemble des documents -comtemporains qui ont servi de base cette lgende de Molire, -excellente consulter, mais qu'il est bon de soumettre l'examen le -plus scrupuleux. diff --git a/tools/retro-library/charset_normalizer/data/sample-french.txt b/tools/retro-library/charset_normalizer/data/sample-french.txt deleted file mode 100644 index 640431000..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-french.txt +++ /dev/null @@ -1,59 +0,0 @@ -JEAN-BAPTISTE POQUELIN MOLIÈRE -NÉ LE 15 JANVIER 1622, MORT LE 17 FÉVRIER 1673 - - -«Quel est le plus grand des écrivains de mon règne? demandait Louis XIV -à Boileau.--Sire, c'est Molière.» - -Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains -que la France a produits, sans excepter Voltaire lui-même, imprégné de -l'esprit anglais par son séjour à Londres, c'est incontestablement -Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la -plus complète le fond du génie français. - -En raison de cette identité de son génie avec le nôtre, il exerça sur -l'époque subséquente, sur le dix-huitième siècle, sur l'époque même où -nous écrivons, la plus active, la plus redoutable influence. Tout ce -qu'il a voulu détruire est en ruine. Les types qu'il a créés ne peuvent -mourir. Le sens de la vie pratique, qu'il a recommandé d'après Gassendi, -a fini par l'emporter sur les idées qui imposaient à la société -française. Il n'y a pas de superstition qu'il n'ait attaquée, pas de -crédulité qu'il n'ait saisie corps à corps pour la terrasser, pas de -formule qu'il ne se soit efforcé de détruire. A-t-il, comme l'exprime si -bien Swift, _déchiré l'étoffe avec la doublure_? l'histoire le dira. Ce -qui est certain, c'est que l'élève de Lucrèce, le protégé de Louis XIV, -poursuivait un but déterminé vers lequel il a marché d'un pas ferme, -obstiné, tantôt foulant aux pieds les obstacles, tantôt les tournant -avec adresse. Le sujet de _Tartuffe_ est dans Lucrèce; à Lucrèce -appartient ce vers, véritable devise de Molière: - - _Et religionis..... nodos solvere curo[1]._ - -La puissance de Molière sur les esprits a été telle, qu'une légende -inexacte, calomnieuse de son vivant, romanesque après sa mort, s'est -formée autour de cette gloire populaire. Il est un mythe comme Jules -César et Apollon. - - [1] Ce que je veux, c'est rompre les entraves qui nous enchaînent - (_religionis.... quod religat_). - -Dates, événements, réalités, souvenirs, sont venus se confondre dans un -inextricable chaos où la figure de Molière a disparu. Tous les vices -jusqu'à l'ivrognerie, jusqu'à l'inceste et au vol, lui furent imputés de -son vivant. Les vertus les plus éthérées lui furent attribuées par les -prêtres de son culte. Homme d'action, sans cesse en face du public, du -roi ou de sa troupe, occupé de son gouvernement et de la création de ses -œuvres, il n'a laissé aucune trace de sa propre vie, aucun document -biographique, à peine une lettre. Les pamphlets pour et contre lui -composaient déjà une bibliothèque, lorsqu'un écouteur aux portes, nommé -Grimarest, collecteur d'anas, aimant l'exagération des récits et -incapable de critique, prétendit, trente-deux ans après la mort du -comédien populaire, raconter et expliquer sa vie. Vers la même époque, -une comédienne, à ce que l'on croit du moins, forcée de se réfugier en -Hollande, jetait dans un libelle les souvenirs de coulisse qu'elle avait -pu recueillir sur l'intérieur du ménage de Molière et de sa femme. Enfin -quelques détails authentiques, semés dans l'édition de ses œuvres -publiée par Lagrange en 1682, complètent l'ensemble des documents -comtemporains qui ont servi de base à cette légende de Molière, -excellente à consulter, mais qu'il est bon de soumettre à l'examen le -plus scrupuleux. diff --git a/tools/retro-library/charset_normalizer/data/sample-greek-2.txt b/tools/retro-library/charset_normalizer/data/sample-greek-2.txt deleted file mode 100644 index bfddcfaca..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-greek-2.txt +++ /dev/null @@ -1 +0,0 @@ - 12 , . , , . - , , , , , . 20 . diff --git a/tools/retro-library/charset_normalizer/data/sample-greek.txt b/tools/retro-library/charset_normalizer/data/sample-greek.txt deleted file mode 100644 index bfddcfaca..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-greek.txt +++ /dev/null @@ -1 +0,0 @@ - 12 , . , , . - , , , , , . 20 . diff --git a/tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt b/tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt deleted file mode 100644 index ea804346f..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-hebrew-2.txt +++ /dev/null @@ -1 +0,0 @@ - . , . (), , . (), (). diff --git a/tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt b/tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt deleted file mode 100755 index ea804346f..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-hebrew-3.txt +++ /dev/null @@ -1 +0,0 @@ - . , . (), , . (), (). diff --git a/tools/retro-library/charset_normalizer/data/sample-korean.txt b/tools/retro-library/charset_normalizer/data/sample-korean.txt deleted file mode 100644 index 6f632b190..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-korean.txt +++ /dev/null @@ -1 +0,0 @@ - ڵ 絵 ż ƴ, ΰμ ߱Ѵ. 20 Ŀ , 縯 ȯ ̴ ũλ ̳  ǰ ִ. ѹα ѱ ⵶ ( ؿ) ũλ Ͽ, Ұϰ ִ. diff --git a/tools/retro-library/charset_normalizer/data/sample-polish.txt b/tools/retro-library/charset_normalizer/data/sample-polish.txt deleted file mode 100644 index 9e506c26d..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-polish.txt +++ /dev/null @@ -1,204 +0,0 @@ -"source";"target" -"REF.-2";"POLISH" -"KW-P00-01";"SYSTEM VIDEODOMOFONOWY MEET" -"KW-P00-02";"URZĄDZENIE" -"KW-P00-03";"OGÓLNE" -"KW-P00-04";"SIEĆ" -"KW-P00-05";"KD" -"KW-P00-06";"ROZP. TWARZY." -"KW-P00-07";"KAMERY IP" -"KW-P00-08";"SIP" -"KW-P00-09";"SIP TRUNK" -"KW-P00-10";"PRZEKIEROWANIA" -"KW-P00-11";"ZAAWANSOWANE" -"KW-P00-12";"KOD PIN" -"KW-P00-13";"WECHAT QR" -"KW-P00-14";"PRZYWRACAĆ" -"KW-P00-16";"WINDA" -"KW-P01-01";"INFORMACJE O URZĄDZENIU" -"KW-P01-02";"PANEL VIDEO FOOBAR KIN" -"KW-P01-03";"FIRMWARE: V02.10" -"KW-P01-04";"URZĄDZENIE: PANEL BLOKOWY-CYFROWY 001-02" -"KW-P01-05";"URZĄDZENIE: PANEL BLOKOWY PRZYCISKI 020-02" -"KW-P01-06";"URZĄDZENIE: PANEL GŁÓWNY 01" -"KW-P01-07";"URZĄDZENIE: PANEL 1W 006-0102-01" -"KW-P01-08";"NUMER SERYJNY:" -"KW-P01-09";"MAC:" -"KW-P01-10";"IP:" -"KW-P01-11";"COPYRIGHT © FOOBAR " -"KW-P01-12";"www.example.com" -"KW-P02-01";"USTAWIENIA GŁÓWNE" -"KW-P02-02";"TYP:" -"KW-P02-03";"PANEL GŁÓWNY" -"KW-P02-04";"CYFROWY P. BLOKOWY" -"KW-P02-05";"P. BLOK. PRZYCISKI" -"KW-P02-06";"PANEL 1NR" -"KW-P02-07";"BLOK:" -"KW-P02-08";"LOKAL:" -"KW-P02-09";"MONIT WYŚWIETLACZA:" -"KW-P02-10";"THIS INTERFACE IS NOT ENABLED" -"KW-P02-11";"NUMER PANELU:" -"KW-P02-12";"NAZWA URZĄDZENIA:" -"KW-P02-13";"(≤16 ZNAKÓW)" -"KW-P02-14";"JĘZYK:" -"KW-P02-15";"ENGLISH" -"KW-P02-16";"中文" -"KW-P02-17";"ESPAÑOL" -"KW-P02-18";"РУССКИЙ" -"KW-P02-19";"DEUTSCH" -"KW-P02-20";"TÜRKÇE" -"KW-P02-21";"POLSKI" -"KW-P02-22";"עברית" -"KW-P02-23";"FRANÇAIS" -"KW-P02-24";"فارسی" -"KW-P02-25";"GŁOŚNOŚĆ PANELU:" -"KW-P02-26";"JASNOŚĆ" -"KW-P02-27";"ROZDZIELCZOŚĆ VIDEO:" -"KW-P02-28";"TRYB PRZEKIEROWANIA SIP:" -"KW-P02-29";"SEKWENCYJNE" -"KW-P02-30";"JEDNOCZESNE" -"KW-P02-31";"PORTIER:" -"KW-P02-32";"PORTIERNIA 1:" -"KW-P02-33";"PORTIERNIA 2:" -"KW-P02-34";"USTAW. DATY I CZASU" -"KW-P02-35";"FORMAT DATY:" -"KW-P02-36";"DATA:" -"KW-P02-37";"CZAS:" -"KW-P02-38";"STREFA CZASOWA:" -"KW-P02-39";"ZAPISZ" -"KW-P02-40";"BŁĘDNE DANE" -"KW-P02-41";"KLAWIATURA ALFANUM.:" -"KW-P02-42";"KOMUNIKAT OTWARCIA DRZWI:" -"KW-P02-43";"WYGASZACZ EKRANU:" -"KW-P02-44";"WSPARCIE:" -"KW-P02-45";"OCZEKIWANIE" -"KW-P02-46";"POŁĄCZENIE" -"KW-P02-47";"WSPARCIE" -"KW-P02-48";"lista" -"KW-P02-49";"DST:" -"KW-P02-57";"TŁO:" -"KW-P02-58";"CIEMNE" -"KW-P02-59";"JASNE" -"KW-P02-60";"IMPORT" -"KW-P02-61";"EKSPORT" -"KW-P02-62";"USUŃ" -"KW-P02-63";"WYBIERZ PRAWIDŁOWY PLIK PNG" -"KW-P02-64";"IMPORTUJ" -"KW-P02-65";"WYSYŁANIE ZAKOŃCZONE" -"KW-P02-66";"BRAK OBRAZU" -"KW-P02-67";"USUNIĘTE" -"KW-P02-68";"BŁĄD USUWANIA" -"KW-P03-01";"USTAWIENIA SIECI" -"KW-P03-02";"IP:" -"KW-P03-03";"MASKA:" -"KW-P03-04";"BRAMA:" -"KW-P03-05";"DNS:" -"KW-P03-06";"SOFTWARE IP:" -"KW-P03-07";"SW. PIN:" -"KW-P03-08";"ZAPISZ" -"KW-P04-01";"USTAWIENIA KONTROLI DOSTĘPU" -"KW-P04-02";"PRZYCISK EGRESS:" -"KW-P04-03";"CZAS ELEKTROZACZEPU:" -"KW-P04-04";"CZAS KONTAKTRONU:" -"KW-P04-05";"REF.1491 4 RELAY:" -"KW-P04-06";"CZAS ELEKTROZACZEPU:" -"KW-P04-07";"CZAS KONTAKTRONU:" -"KW-P04-08";"KARTA ADMINISTRATORA:" -"KW-P04-09";"ROZBRAJANIE KARTĄ:" -"KW-P04-10";"MONITY KART:" -"KW-P04-11";"KOD GOŚCIA:" -"KW-P04-12";"KOD DOSTĘPU:" -"KW-P04-13";"#1" -"KW-P04-14";"#2" -"KW-P04-15";"#3" -"KW-P04-16";"#4" -"KW-P04-17";"ALARM DRZWI" -"KW-P04-18";"GWAŁTOWNY ALARM OTWARCIA" -"KW-P04-19";"WIEGAND:" -"KW-P04-20";"BURST" -"KW-P04-21";"26-BIT" -"KW-P04-22";"FACILITY:" -"KW-P04-24";"ZAPISZ" -"KW-P04-25";"WYŁĄCZONY" -"KW-P04-26";"REF.1490 2 RELAY:" -"KW-P04-27";"KOD QR:" -"KW-P04-28";"WIEGAND:" -"KW-P04-29";"26-BIT" -"KW-P04-30";"34-BIT" -"KW-P04-31";"KOD MIEJSCA:" -"KW-P04-32";"AUTO AKTYWACJA:" -"KW-P04-33";"BŁĘDNE DANE" -"KW-P05-01";"ROZPOZNAWANIE TWARZY" -"KW-P05-02";"ROZPOZNAWANIE TWARZY:" -"KW-P05-04";"MODEL:" -"KW-P05-05";"Wykrycie obecności:" -"KW-P05-06";"WŁĄCZONY" -"KW-P05-07";"WYŁĄCZONY" -"KW-P05-08";"PODOBIEŃSTWO:" -"KW-P05-09";"NISKIE" -"KW-P05-10";"ŚREDNIE" -"KW-P05-11";"WYSOKIE" -"KW-P05-12";"ZAPISZ" -"KW-P06-01";"USTAWIENIA KAMER IP" -"KW-P06-02";"ILOŚĆ KAMER:" -"KW-P06-03";"KAMERA" -"KW-P06-04";"URL:" -"KW-P06-05";"ZAPISZ" -"KW-P07-01";"USTAWIENIA SIP" -"KW-P07-02";"WŁĄCZ SIP:" -"KW-P07-03";"SPRAWDŹ STATUS SIP" -"KW-P07-04";"SIP ZAREJESTROWANY" -"KW-P07-05";"BŁĄD REJESTRACJI SIP" -"KW-P07-06";"SERWER SIP:" -"KW-P07-07";"DOMENA:" -"KW-P07-08";"OUTBOUND:" -"KW-P07-09";"STUN IP:" -"KW-P07-10";"PORT STUN:" -"KW-P07-11";"H.264:" -"KW-P07-12";"UŻYTKOWNIK SIP:" -"KW-P07-13";"HASŁO SIP:" -"KW-P07-14";"CZAS ROZMOWY:" -"KW-P07-15";"CZAS DZWONIENIA:" -"KW-P07-16";"ZAPISZ" -"KW-P08-01";"USTAWIENIA SIP TRUNK" -"KW-P08-02";"WŁĄCZ SIP TRUNK:" -"KW-P08-03";"URL:" -"KW-P08-04";"ZAPISZ" -"KW-P09-01";"USTAWIENIA PRZEKIEROWAŃ" -"KW-P09-02";"IMPORT" -"KW-P09-03";"EKSPORT" -"KW-P09-04";"APARTAMENT" -"KW-P09-05";"NUMER" -"KW-P10-01";"USTAWIENIA ZAAWANSOWANE" -"KW-P10-02";"SZYBKIE WYBIERANIE:" -"KW-P10-03";"URL:" -"KW-P10-04";"ONU:" -"KW-P10-05";"MAPOWANIE POŁĄCZEŃ:" -"KW-P10-06";"BIAŁA LISTA:" -"KW-P10-07";"Lista telefoniczna:" -"KW-P10-08";"IMPORT" -"KW-P10-09";"EKSPORT" -"KW-P10-10";"IMPORTUJ" -"KW-P10-11";"WYSYŁANIE ZAKOŃCZONE" -"KW-P10-12";"UŻYJ WŁAŚCIWEGO PLIKU CSV." -"KW-P10-13";"OK" -"KW-P10-14";"ZAPISZ" -"KW-P11-01";"USTAWIENIA KODU PIN" -"KW-P11-02";"OBECNY PIN:" -"KW-P11-03";"NOWY PIN:" -"KW-P11-04";"POTWIERDŹ PIN:" -"KW-P11-05";"ZAPISZ" -"KW-P12-01";"WECHAT QR" -"KW-P12-02";"WŁĄCZ" -"KW-P12-03";"UUID:" -"KW-P12-04";"HASŁO:" -"KW-P12-05";"SERWER:" -"KW-P12-06";"WŁĄCZ CZYTNIK QR:" -"KW-P12-07";"STATUS:" -"KW-P12-08";"REJESTRACJA POMYŚLNIE" -"KW-P12-09";"REJESTRACJA NIE POWIODŁA SIĘ" -"KW-P12-10";"ZAPISZ" -"KW-P13-01";"PRZYWRACAĆ" -"KW-P13-02";"PRZYWRÓCIĆ USTAWIENIA FABRYCZNE" -"KW-P13-03";"POTWIERDZAĆ PRZYWRÓĆ USTAWIENIA FABRYCZNE?" -"KW-P13-04";"URZĄDZENIE REBOOT" diff --git a/tools/retro-library/charset_normalizer/data/sample-russian-2.txt b/tools/retro-library/charset_normalizer/data/sample-russian-2.txt deleted file mode 100644 index 79cf59863..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-russian-2.txt +++ /dev/null @@ -1,5 +0,0 @@ -В гимназии он не был в числе первых учеников (исключение составляли математика и латынь). Укоренившаяся система механического заучивания материала учащимися (которая, как он считал, наносит вред самому духу учёбы и творческому мышлению), а также авторитарное отношение учителей к ученикам вызывало у Альберта Эйнштейна неприятие, поэтому он часто вступал в споры со своими преподавателями. - -После окончательного разорения отца семейства в 1894 году Эйнштейны переехали из Мюнхена в итальянский город Павию, близ Милана. Сам Альберт оставался в Мюнхене ещё некоторое время, чтобы окончить все шесть классов гимназии. Так и не получив аттестата зрелости, в 1895 году он присоединился к своей семье в Милане. - -Осенью 1895 г. Альберт Эйнштейн прибыл в Швейцарию, чтобы сдать вступительные экзамены в Высшее техническое училище (Политехникум) в Цюрихе и стать преподавателем физики. Блестяще проявив себя на экзамене по математике, он в то же время провалил экзамены по ботанике и французскому языку, что не позволило ему поступить в Цюрихский Политехникум. Однако директор училища посоветовал молодому человеку поступить в выпускной класс школы в Аарау (Швейцария), чтобы получить аттестат и повторить поступление. diff --git a/tools/retro-library/charset_normalizer/data/sample-russian-3.txt b/tools/retro-library/charset_normalizer/data/sample-russian-3.txt deleted file mode 100644 index fbf2db067..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-russian-3.txt +++ /dev/null @@ -1,7 +0,0 @@ -Москва́ (произношение (инф.)) — столица России, город федерального значения, административный центр Центрального федерального округа и центр Московской области, в состав которой не входит[6]. Крупнейший по численности населения город России и её субъект — 12 655 050[3] человек (2021), самый населённый из городов, полностью расположенных в Европе, занимает 22 место среди городов мира по численности населения[7], крупнейший русскоязычный город в мире. Центр Московской городской агломерации. - -Историческая столица Великого княжества Московского, Русского царства, Российской империи (в 1728—1732 годах[8][9][10][11]), Советской России и СССР. Город-герой. В Москве находятся федеральные органы государственной власти Российской Федерации (за исключением Конституционного суда), посольства иностранных государств, штаб-квартиры большинства крупнейших российских коммерческих организаций и общественных объединений. - -Расположена на западе России, на реке Москве в центре Восточно-Европейской равнины, в междуречье Оки и Волги. Как субъект федерации, Москва граничит с Московской и Калужской областями. - -Москва — популярный туристический центр России. Кремль, Красная площадь, Новодевичий монастырь и Церковь Вознесения в Коломенском входят в список объектов всемирного наследия ЮНЕСКО[12]. Она является важнейшим транспортным узлом: город обслуживают 6 аэропортов, 10 железнодорожных вокзалов, 3 речных порта (имеется речное сообщение с морями бассейнов Атлантического и Северного Ледовитого океанов). С 1935 года в Москве работает метрополитен. Москва — спортивный центр страны. В 1980 году в Москве прошли XXII летние Олимпийские игры, а в 2018 город стал одним из хозяев чемпионата мира по футболу. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/data/sample-russian.txt b/tools/retro-library/charset_normalizer/data/sample-russian.txt deleted file mode 100644 index 5a5e77c0b..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-russian.txt +++ /dev/null @@ -1,5 +0,0 @@ - ( ). (, , ), , . - - 1894 , . , . , 1895 . - - 1895 . , () . , , . (), . diff --git a/tools/retro-library/charset_normalizer/data/sample-spanish.txt b/tools/retro-library/charset_normalizer/data/sample-spanish.txt deleted file mode 100755 index 9b9aadcc8..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-spanish.txt +++ /dev/null @@ -1,33 +0,0 @@ -La creación - -1 En el principio creó Dios los cielos y la tierra. 2 Y la tierra estaba sin orden y vacía[a], y las tinieblas cubrían la superficie[b] del abismo, y el Espíritu de Dios se movía sobre la superficie[c] de las aguas. 3 Entonces dijo Dios: Sea la luz. Y hubo luz. 4 Y vio Dios que la luz era buena; y separó Dios la luz de las tinieblas. 5 Y llamó Dios a la luz día, y a las tinieblas llamó noche. Y fue la tarde y fue la mañana: un día. - -6 Entonces dijo Dios: Haya expansión[d] en medio de las aguas, y separe las aguas de las aguas. 7 E hizo Dios la expansión, y separó las aguas que estaban debajo de la expansión de las aguas que estaban sobre la expansión. Y fue así. 8 Y llamó Dios a la expansión cielos. Y fue la tarde y fue la mañana: el segundo día. - -9 Entonces dijo Dios: Júntense en un lugar las aguas que están debajo de los cielos, y que aparezca lo seco. Y fue así. 10 Y llamó Dios a lo seco tierra, y al conjunto de las aguas llamó mares. Y vio Dios que era bueno. 11 Y dijo Dios: Produzca la tierra vegetación[e]: hierbas[f] que den semilla, y árboles frutales que den fruto sobre la tierra según su género[g], con su semilla en él. Y fue así. 12 Y produjo la tierra vegetación[h]: hierbas[i] que dan semilla según su género, y árboles que dan fruto con su semilla en él, según su género. Y vio Dios que era bueno. 13 Y fue la tarde y fue la mañana: el tercer día. - -14 Entonces dijo Dios: Haya lumbreras[j] en la expansión de los cielos para separar el día de la noche, y sean para señales y para estaciones y para días y para años; 15 y sean por luminarias en la expansión de los cielos para alumbrar sobre la tierra. Y fue así. 16 E hizo Dios las dos grandes lumbreras[k], la lumbrera[l] mayor para dominio del día y la lumbrera[m] menor para dominio de la noche; hizo también las estrellas. 17 Y Dios las puso en la expansión de los cielos para alumbrar sobre la tierra, 18 y para dominar en el día y en la noche, y para separar la luz de las tinieblas. Y vio Dios que era bueno. 19 Y fue la tarde y fue la mañana: el cuarto día. - -20 Entonces dijo Dios: Llénense[n] las aguas de multitudes de seres vivientes, y vuelen las aves sobre la tierra en la abierta[o] expansión de los cielos. 21 Y creó Dios los grandes monstruos marinos y todo ser viviente que se mueve, de los cuales están llenas[p] las aguas según su género, y toda ave[q] según su género. Y vio Dios que era bueno. 22 Y Dios los bendijo, diciendo: Sed fecundos y multiplicaos, y llenad las aguas en los mares, y multiplíquense las aves en la tierra. 23 Y fue la tarde y fue la mañana: el quinto día. - -24 Entonces dijo Dios: Produzca la tierra seres vivientes según su género: ganados, reptiles y bestias de la tierra según su género. Y fue así. 25 E hizo Dios las bestias de la tierra según su género, y el ganado según su género, y todo lo que se arrastra sobre la tierra según su género. Y vio Dios que era bueno. -Creación del hombre y de la mujer - -26 Y dijo Dios: Hagamos al hombre a nuestra imagen, conforme a nuestra semejanza; y ejerza[r] dominio sobre los peces del mar, sobre las aves del cielo, sobre los ganados, sobre toda la tierra, y sobre todo reptil que se arrastra sobre la tierra. 27 Creó, pues, Dios al hombre a imagen suya, a imagen de Dios lo creó; varón y hembra los creó. 28 Y los bendijo Dios y les dijo[s]: Sed fecundos y multiplicaos, y llenad la tierra y sojuzgadla; ejerced dominio sobre los peces del mar, sobre las aves del cielo y sobre todo ser viviente que se mueve[t] sobre la tierra. 29 Y dijo Dios: He aquí, yo os he dado toda planta que da semilla que hay en la superficie[u] de toda la tierra, y todo árbol que tiene fruto[v] que da semilla; esto os servirá de[w] alimento. 30 Y a toda bestia de la tierra, a toda ave de los cielos y a todo lo que se mueve[x] sobre la tierra, y que tiene vida[y], les he dado toda planta verde para alimento. Y fue así. 31 Y vio Dios todo lo que había hecho, y he aquí que era bueno en gran manera. Y fue la tarde y fue la mañana: el sexto día. - -Así fueron acabados los cielos y la tierra y todas sus huestes. 2 Y en el séptimo día completó Dios la[a] obra que había hecho, y reposó en el día séptimo de toda la[b] obra que había hecho. 3 Y bendijo Dios el séptimo día y lo santificó, porque en él reposó de toda la[c] obra que El[d] había creado y hecho[e]. -El huerto del Edén - -4 Estos son los orígenes[f] de los cielos y de la tierra cuando fueron creados, el día en que el Señor Dios hizo la tierra y los cielos. 5 Y aún no había ningún arbusto del campo en la tierra, ni había aún brotado ninguna planta[g] del campo, porque el Señor Dios no había enviado lluvia sobre la tierra, ni había hombre para labrar[h] la tierra. 6 Pero se levantaba de la tierra un vapor[i] que regaba toda la superficie[j] del suelo. 7 Entonces el Señor Dios formó al hombre del polvo de la tierra, y sopló en su nariz el aliento de vida; y fue el hombre un ser[k] viviente. 8 Y plantó el Señor Dios un huerto hacia el oriente, en Edén; y puso allí al hombre que había formado. 9 Y el Señor Dios hizo brotar de la tierra todo árbol agradable a la vista y bueno para comer; asimismo, en medio del huerto, el árbol de la vida y el árbol del conocimiento[l] del bien y del mal. - -10 Y del Edén salía un río para regar el huerto, y de allí se dividía y se convertía en otros cuatro ríos[m]. 11 El nombre del primero es Pisón; éste es el que rodea toda la tierra de Havila, donde hay oro. 12 El oro de aquella tierra es bueno; allí hay bedelio y ónice. 13 Y el nombre del segundo río es Gihón; éste es el que rodea la tierra de Cus. 14 Y el nombre del tercer río es Tigris[n]; éste es el que corre[o] al oriente de Asiria. Y el cuarto río es el Eufrates[p]. 15 Entonces el Señor Dios tomó al hombre y lo puso en el huerto del Edén, para que lo cultivara y lo cuidara. 16 Y ordenó el Señor Dios al hombre, diciendo: De todo árbol del huerto podrás comer, 17 pero del árbol del conocimiento[q] del bien y del mal no comerás[r], porque el día que de él comas, ciertamente morirás. -Formación de la mujer - -18 Y el Señor Dios dijo: No es bueno que el hombre esté solo; le haré una ayuda idónea[s]. 19 Y el Señor Dios formó de la tierra todo animal del campo y toda ave del cielo, y los trajo al hombre para ver cómo los llamaría; y como el hombre llamó a cada ser viviente, ése fue su nombre. 20 Y el hombre puso nombre a todo ganado y a las aves del cielo y a toda bestia del campo, mas para Adán[t] no se encontró una ayuda que fuera idónea para él[u]. 21 Entonces el Señor Dios hizo caer un sueño profundo sobre el hombre, y éste se durmió; y Dios tomó una de sus costillas, y cerró la carne en ese lugar. 22 Y de la costilla que el Señor Dios había tomado del hombre, formó[v] una mujer y la trajo al hombre. 23 Y el hombre dijo: - -Esta es ahora hueso de mis huesos, -y carne de mi carne; -ella[w] será llamada mujer[x], -porque del hombre[y] fue tomada. - -24 Por tanto el hombre dejará a su padre y a su madre y se unirá a su mujer, y serán una sola carne. 25 Y estaban ambos desnudos, el hombre y su mujer, y no se avergonzaban. \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/data/sample-turkish.txt b/tools/retro-library/charset_normalizer/data/sample-turkish.txt deleted file mode 100644 index c86234301..000000000 --- a/tools/retro-library/charset_normalizer/data/sample-turkish.txt +++ /dev/null @@ -1,33 +0,0 @@ -stanbul, Trkiye'nin en kalabalk, iktisadi ve kltrel adan en nemli -ehri.[2][3][4] ktisadi byklk adan dnyada 34., nfus asndan -belediye snrlar gz nne alnarak yaplan sralamaya gre Avrupa'da -birinci srada gelir.[5][6] - - -stanbul Trkiye'nin kuzeybatsnda, Marmara kys ve Boazii boyunca, -Hali'i de evreleyecek ekilde kurulmutur.[7] stanbul ktalararas bir -ehir olup, Avrupa'daki blmne Avrupa Yakas veya Rumeli Yakas, -Asya'daki blmne ise Anadolu Yakas denir. Tarihte ilk olarak taraf -Marmara Denizi, Boazii ve Hali'in sard bir yarm ada zerinde kurulan -stanbul'un batdaki snrn stanbul Surlar oluturmaktayd. Gelime ve -byme srecinde surlarn her seferinde daha batya ilerletilerek ina -edilmesiyle 4 defa geniletilen ehrin [8] 39 ilesi vardr.[9] Snrlar -ierisinde ise bykehir belediyesi ile birlikte toplam 40 belediye -bulunmaktadr. - - -Dnyann en eski ehirlerinden biri olan stanbul, M.S. 330 - 395 yllar -arasnda Roma mparatorluu, 395 - 1204 ile 1261 - 1453 yllar arasnda -Dou Roma mparatorluu, 1204 - 1261 arasnda Latin mparatorluu ve son -olarak 1453 - 1922 yllar arasnda Osmanl mparatorluu'na bakentlik -yapt.[10] Ayrca, hilafetin Osmanl mparatorluu'na getii 1517'den, -kaldrld 1924'e kadar, stanbul slamiyet'in de merkezi oldu.[11] - -1453 ylnda fetihten sonra, kent Osmanl mparatorluu'nun drdnc -bakenti ilan edilidi ve Kostantiniyye Osmanl mparatorluu tarafndan -kentin resmi ad olarak kullanld ve 1923 ylnda Osmanl -mparatorluunun kne kadar, ou zaman bu ad kullanmda -kald. rnein Osmanl mparatorluu ve mahkemeleri, Kostantiniyye'de -yaymlanan resmi belgelerin kaynan belirtmek iin, "be-Makam- -Dar's-Saltanat- Kostantiniyyet'l-Mahrust'l-Mahmiyye" gibi balklar -kullanlrd.[17] diff --git a/tools/retro-library/charset_normalizer/dev-requirements.txt b/tools/retro-library/charset_normalizer/dev-requirements.txt deleted file mode 100644 index 325adac25..000000000 --- a/tools/retro-library/charset_normalizer/dev-requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -chardet==5.1.0 -pytest-cov==4.1.0 -Flask==2.2.3 -pytest>=7.4.4,<=8.3.3 -requests==2.31.0 -pre-commit -build diff --git a/tools/retro-library/charset_normalizer/docs/Makefile b/tools/retro-library/charset_normalizer/docs/Makefile deleted file mode 100755 index b6888c162..000000000 --- a/tools/retro-library/charset_normalizer/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = python -msphinx -SPHINXPROJ = Charset Normalizer -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/tools/retro-library/charset_normalizer/docs/api.rst b/tools/retro-library/charset_normalizer/docs/api.rst deleted file mode 100644 index 3a6fb1325..000000000 --- a/tools/retro-library/charset_normalizer/docs/api.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. _api: - -Developer Interfaces -==================== - -.. module:: charset_normalizer - -Main Interfaces ---------------- - -Those functions are publicly exposed and are protected through our BC guarantee. - -.. autofunction:: from_bytes -.. autofunction:: from_fp -.. autofunction:: from_path -.. autofunction:: is_binary - -.. autoclass:: charset_normalizer.models.CharsetMatches - :inherited-members: -.. autoclass:: charset_normalizer.models.CharsetMatch - :inherited-members: - -.. autofunction:: detect - -.. autofunction:: charset_normalizer.utils.set_logging_handler - - -Mess Detector -------------- - -.. autofunction:: charset_normalizer.md.mess_ratio - -This library allows you to extend the capabilities of the mess detector by extending the -class `MessDetectorPlugin`. - -.. autoclass:: charset_normalizer.md.MessDetectorPlugin - :inherited-members: - - -.. autofunction:: charset_normalizer.md.is_suspiciously_successive_range - - -Coherence Detector ------------------- - -.. autofunction:: charset_normalizer.cd.coherence_ratio - - -Utilities ---------- - -Some reusable functions used across the project. We do not guarantee the BC in this area. - -.. autofunction:: charset_normalizer.utils.is_accentuated - -.. autofunction:: charset_normalizer.utils.remove_accent - -.. autofunction:: charset_normalizer.utils.unicode_range - -.. autofunction:: charset_normalizer.utils.is_latin - -.. autofunction:: charset_normalizer.utils.is_punctuation - -.. autofunction:: charset_normalizer.utils.is_symbol - -.. autofunction:: charset_normalizer.utils.is_emoticon - -.. autofunction:: charset_normalizer.utils.is_separator - -.. autofunction:: charset_normalizer.utils.is_case_variable - -.. autofunction:: charset_normalizer.utils.is_cjk - -.. autofunction:: charset_normalizer.utils.is_hiragana - -.. autofunction:: charset_normalizer.utils.is_katakana - -.. autofunction:: charset_normalizer.utils.is_hangul - -.. autofunction:: charset_normalizer.utils.is_thai - -.. autofunction:: charset_normalizer.utils.is_unicode_range_secondary - -.. autofunction:: charset_normalizer.utils.any_specified_encoding - -.. autofunction:: charset_normalizer.utils.is_multi_byte_encoding - -.. autofunction:: charset_normalizer.utils.identify_sig_or_bom - -.. autofunction:: charset_normalizer.utils.should_strip_sig_or_bom - -.. autofunction:: charset_normalizer.utils.iana_name - -.. autofunction:: charset_normalizer.utils.range_scan - -.. autofunction:: charset_normalizer.utils.is_cp_similar - - -.. class:: os.PathLike -.. class:: typing.BinaryIO diff --git a/tools/retro-library/charset_normalizer/docs/community/faq.rst b/tools/retro-library/charset_normalizer/docs/community/faq.rst deleted file mode 100644 index d1deff4dd..000000000 --- a/tools/retro-library/charset_normalizer/docs/community/faq.rst +++ /dev/null @@ -1,75 +0,0 @@ -Frequently asked questions -=========================== - - -Is UTF-8 everywhere already? ----------------------------- - -Not really, that is a dangerous assumption. Looking at https://w3techs.com/technologies/overview/character_encoding may -seem like encoding detection is a thing of the past but not really. Solo based on 33k websites, you will find -3,4k responses without predefined encoding. 1,8k websites were not UTF-8, merely half! - -This statistic (w3techs) does not offer any ponderation, so one should not read it as -"I have a 97 % chance of hitting UTF-8 content on HTML content". - -(2021 Top 1000 sites from 80 countries in the world according to Data for SEO) https://github.com/potiuk/test-charset-normalizer - -First of all, neither requests, chardet or charset-normalizer are dedicated to HTML content. -The detection concern every text document, like SubRip Subtitle files for instance. And by my own experiences, I never had -a single database using full utf-8, many translated subtitles are from another era and never updated. - -It is so hard to find any stats at all regarding this matter. Users' usages can be very dispersed, so making -assumptions are unwise. - -The real debate is to state if the detection is an HTTP client matter or not. That is more complicated and not my field. - -Some individuals keep insisting that the *whole* Internet is UTF-8 ready. Those are absolutely wrong and very Europe and North America-centered, -In my humble experience, the countries in the world are very disparate in this evolution. And the Internet is not just about HTML content. -Having a thorough analysis of this is very scary. - -Should I bother using detection? --------------------------------- - -In the last resort, yes. You should use well-established standards, eg. predefined encoding, at all times. -When you are left with no clue, you may use the detector to produce a usable output as fast as possible. - -Is it backward-compatible with Chardet? ---------------------------------------- - -If you use the legacy `detect` function, -Then this change is mostly backward-compatible, exception of a thing: - -- This new library support way more code pages (x3) than its counterpart Chardet. -- Based on the 30-ich charsets that Chardet support, expect roughly 80% BC results - -We do not guarantee this BC exact percentage through time. May vary but not by much. - -Isn't it the same as Chardet? ------------------------------ - -The objective is the same, provide you with the best answer (charset) we can given any sequence of bytes. -The method actually differs. - -We do not "train" anything to build a probe for a specific encoding. In addition to finding any languages (intelligent -design) by some rudimentary statistics (character frequency ordering) we built a mess detector to assist the language -detection. - -Any code page supported by your cPython is supported by charset-normalizer! It is that simple, no need to update the -library. It is as generic as we could do. - -I can't build standalone executable ------------------------------------ - -If you are using ``pyinstaller``, ``py2exe`` or alike, you may be encountering this or close to: - - ModuleNotFoundError: No module named 'charset_normalizer.md__mypyc' - -Why? - -- Your package manager picked up a optimized (for speed purposes) wheel that match your architecture and operating system. -- Finally, the module ``charset_normalizer.md__mypyc`` is imported via binaries and can't be seen using your tool. - -How to remedy? - -If your bundler program support it, set up a hook that implicitly import the hidden module. -Otherwise, follow the guide on how to install the vanilla version of this package. (Section: *Optional speedup extension*) diff --git a/tools/retro-library/charset_normalizer/docs/community/featured.rst b/tools/retro-library/charset_normalizer/docs/community/featured.rst deleted file mode 100644 index a704a0bc6..000000000 --- a/tools/retro-library/charset_normalizer/docs/community/featured.rst +++ /dev/null @@ -1,51 +0,0 @@ -Featured projects -================= - -Did you liked how ``charset-normalizer`` perform? and its quality? -You may be interested in those other project maintained by the same authors. -We aim to serve the opensource community the best and as inclusively as we can, no matter -your level or opinions. - -Niquests --------- - -Started as a simple though.. IE 11 has built-in HTTP/2 support while Requests 2.32 does not! - -Most of our programs that interact with HTTP server are built with ``requests`` and -we aren't likely to switch without a substantial effort. - -We just might die at any moment, no notice whatsoever, knowingly that as a Python developer, -we never interacted with a HTTP/2 over TCP or HTTP/3 over QUIC capable server in 2023... - -.. image:: https://dabuttonfactory.com/button.png?t=Get+Niquests+Now&f=Ubuntu-Bold&ts=26&tc=fff&hp=45&vp=20&c=11&bgt=unicolored&bgc=15d798&be=1 - :target: https://github.com/jawah/niquests - -It is a fork of ``requests`` and no breaking changes are to be expected. We made sure that -your migration is effortless and safe. - -httpie-next ------------ - -Easy solution are cool, let us introduce you to HTTPie but with built-in support -for HTTP/2 and HTTP/3. -It is made available as a plugin, no effort required beside installing the said plugin. - -Enjoy HTTPie refreshed! - -.. image:: https://dabuttonfactory.com/button.png?t=Get+HTTPie-Next+Now&f=Ubuntu-Bold&ts=26&tc=fff&hp=45&vp=20&c=11&bgt=unicolored&bgc=15d798&be=1 - :target: https://github.com/Ousret/httpie-next - -Wassima -------- - -Did you ever wonder how would it feel like to leave the headache with root CAs (certificate authority)? -Well, you may, starting today, use your operating system trusted root CAs to verify -peer certificates with the at most comfort. - -It is enabled by default in Niquests, but you can use that awesome feature by itself. - -.. image:: https://dabuttonfactory.com/button.png?t=OS+root+CAs+for+Python&f=Ubuntu-Bold&ts=26&tc=fff&hp=45&vp=20&c=11&bgt=unicolored&bgc=15d798&be=1 - :target: https://github.com/jawah/wassima - -The solution is universal and served for (almost) every possible case. -You may remove the certifi package, let it rest in peace. diff --git a/tools/retro-library/charset_normalizer/docs/community/speedup.rst b/tools/retro-library/charset_normalizer/docs/community/speedup.rst deleted file mode 100644 index 1e65379ac..000000000 --- a/tools/retro-library/charset_normalizer/docs/community/speedup.rst +++ /dev/null @@ -1,50 +0,0 @@ -Optional speedup extension -========================== - -Why? ----- - -charset-normalizer will always remain pure Python, meaning that a environment without any build capabilities will -run this program without any additional requirements. - -Nonetheless, starting from the version 3.0 we introduce and publish some platform specific wheels including a -pre-built extension. - -Most of the time is spent in the module `md.py` so we decided to "compile it" using Mypyc. - -(1) It does not require to have a separate code base -(2) Our project code base is rather simple and lightweight -(3) Mypyc is robust enough today -(4) Four times faster! - -How? ----- - -If your platform and/or architecture is not served by this swift optimization you may compile it easily yourself. -Following those instructions (provided you have the necessary toolchain installed): - - :: - - export CHARSET_NORMALIZER_USE_MYPYC=1 - pip install mypy build wheel - pip install charset-normalizer --no-binary :all: - - -How not to? ------------ - -You may install charset-normalizer without the speedups by directly using the universal wheel -(most likely hosted on PyPI or any valid mirror you use) with ``--no-binary``. - -E.g. when installing ``requests`` and you don't want to use the ``charset-normalizer`` speedups, you can do: - - :: - - pip install requests --no-binary charset-normalizer - - -When installing `charset-normalizer` by itself, you can also pass ``:all:`` as the specifier to ``--no-binary``. - - :: - - pip install charset-normalizer --no-binary :all: diff --git a/tools/retro-library/charset_normalizer/docs/community/why_migrate.rst b/tools/retro-library/charset_normalizer/docs/community/why_migrate.rst deleted file mode 100644 index 1909c7705..000000000 --- a/tools/retro-library/charset_normalizer/docs/community/why_migrate.rst +++ /dev/null @@ -1,18 +0,0 @@ -Why should I migrate to Charset-Normalizer? -=========================================== - -There is so many reason to migrate your current project. Here are some of them: - -- Remove ANY license ambiguity/restriction for projects bundling Chardet (even indirectly). -- X10 faster than Chardet in average and X6 faster in 99% of the cases AND support 3 times more encoding. -- Never return a encoding if not suited for the given decoder. Eg. Never get UnicodeDecodeError! -- Actively maintained, open to contributors. -- Have the backward compatible function ``detect`` that come from Chardet. -- Truly detect the language used in the text. -- It is, for the first time, really universal! As there is no specific probe per charset. -- The package size is X2~X4 lower than Chardet's (5.0)! (Depends on your arch) -- Propose much more options/public kwargs to tweak the detection as you sees fit! -- Using static typing to ease your development. -- Detect Unicode content better than Chardet or cChardet does. - -And much more..! What are you waiting for? Upgrade now and give us a feedback. (Even if negative) diff --git a/tools/retro-library/charset_normalizer/docs/conf.py b/tools/retro-library/charset_normalizer/docs/conf.py deleted file mode 100755 index 46510fade..000000000 --- a/tools/retro-library/charset_normalizer/docs/conf.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# charset-normalizer documentation build configuration file, created by -# sphinx-quickstart on Fri Jun 16 04:30:35 2017. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -import sys -import os - -sys.path.insert(0, os.path.abspath("..")) - -import charset_normalizer - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -# source_suffix = '.rst' - -source_parsers = {} - -source_suffix = ['.rst',] - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = 'charset_normalizer' -copyright = '2023, Ahmed TAHRI' -author = 'Ahmed TAHRI' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = charset_normalizer.__version__ -# The full version, including alpha/beta/rc tags. -release = charset_normalizer.__version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = "en" - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'furo' - -html_theme_path = [] - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'charset-normalizer-doc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'charset-normalizer.tex', 'Charset Normalizer Documentation', - 'Ahmed TAHRI', 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'charset-normalizer', 'Charset Normalizer Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -texinfo_documents = [ - (master_doc, 'Charset Normalizer', 'Charsert Normalizer Documentation', - author, 'charset-normalizer', '🔎 Like Chardet. 🚀 Package for encoding & language detection. Charset detection.', - 'Miscellaneous'), -] diff --git a/tools/retro-library/charset_normalizer/docs/index.rst b/tools/retro-library/charset_normalizer/docs/index.rst deleted file mode 100755 index 19ca08a90..000000000 --- a/tools/retro-library/charset_normalizer/docs/index.rst +++ /dev/null @@ -1,93 +0,0 @@ -=================== - Charset Normalizer -=================== - -Overview -======== - -A Library that helps you read text from unknown charset encoding. -This project is motivated by chardet, I'm trying to resolve the issue by taking another approach. -All IANA character set names for which the Python core library provides codecs are supported. - -It aims to be as generic as possible. - -.. image:: https://repository-images.githubusercontent.com/200259335/d3da9600-dedc-11e9-83e8-081f597505df - :width: 500px - :alt: CLI Charset Normalizer - :align: right - - -It is released under MIT license, see LICENSE for more -details. Be aware that no warranty of any kind is provided with this package. - -Copyright (C) 2023 Ahmed TAHRI - -Introduction -============ - -This library aim to assist you in finding what encoding suit the best to content. -It **DOES NOT** try to uncover the originating encoding, in fact this program does not care about it. - -By originating we means the one that was precisely used to encode a text file. - -Precisely :: - - my_byte_str = 'Bonjour, je suis à la recherche d\'une aide sur les étoiles'.encode('cp1252') - - -We **ARE NOT** looking for cp1252 **BUT FOR** ``Bonjour, je suis à la recherche d'une aide sur les étoiles``. -Because of this :: - - my_byte_str.decode('cp1252') == my_byte_str.decode('cp1256') == my_byte_str.decode('cp1258') == my_byte_str.decode('iso8859_14') - # Print True ! - -There is no wrong answer to decode ``my_byte_str`` to get the exact same result. -This is where this library differ from others. There's not specific probe per encoding table. - -Features -======== - -- Encoding detection on a fp (file pointer), bytes or PathLike. -- Transpose any encoded content to Unicode the best we can. -- Detect spoken language in text. -- Ship with a great CLI. -- Also, detect binaries. - -Start Guide ------------ - -.. toctree:: - :maxdepth: 2 - - user/support - user/getstarted - user/advanced_search - user/handling_result - user/miscellaneous - user/cli - -Community Guide ---------------- - -.. toctree:: - :maxdepth: 2 - - community/speedup - community/faq - community/why_migrate - community/featured - -Developer Guide ---------------- - -.. toctree:: - :maxdepth: 3 - - api - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/tools/retro-library/charset_normalizer/docs/make.bat b/tools/retro-library/charset_normalizer/docs/make.bat deleted file mode 100644 index 4be663f46..000000000 --- a/tools/retro-library/charset_normalizer/docs/make.bat +++ /dev/null @@ -1,36 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=build -set SPHINXPROJ=charset_normalizer - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/tools/retro-library/charset_normalizer/docs/requirements.txt b/tools/retro-library/charset_normalizer/docs/requirements.txt deleted file mode 100755 index de8fca92b..000000000 --- a/tools/retro-library/charset_normalizer/docs/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -Sphinx -furo diff --git a/tools/retro-library/charset_normalizer/docs/user/advanced_search.rst b/tools/retro-library/charset_normalizer/docs/user/advanced_search.rst deleted file mode 100644 index a269cd102..000000000 --- a/tools/retro-library/charset_normalizer/docs/user/advanced_search.rst +++ /dev/null @@ -1,82 +0,0 @@ -Advanced Search -=============== - -Charset Normalizer method ``from_bytes``, ``from_fp`` and ``from_path`` provide some -optional parameters that can be tweaked. - -As follow :: - - from charset_normalizer import from_bytes - - my_byte_str = 'Bсеки човек има право на образование.'.encode('cp1251') - - results = from_bytes( - my_byte_str, - steps=10, # Number of steps/block to extract from my_byte_str - chunk_size=512, # Set block size of each extraction - threshold=0.2, # Maximum amount of chaos allowed on first pass - cp_isolation=None, # Finite list of encoding to use when searching for a match - cp_exclusion=None, # Finite list of encoding to avoid when searching for a match - preemptive_behaviour=True, # Determine if we should look into my_byte_str (ASCII-Mode) for pre-defined encoding - explain=False, # Print on screen what is happening when searching for a match - language_threshold=0.1 # Minimum coherence ratio / language ratio match accepted - ) - - -Using CharsetMatches ------------------------------- - -Here, ``results`` is a ``CharsetMatches`` object. It behave like a list but does not implements all related methods. -Initially, it is sorted. Calling ``best()`` is sufficient to extract the most probable result. - -.. autoclass:: charset_normalizer.CharsetMatches - :members: - -List behaviour --------------- - -Like said earlier, ``CharsetMatches`` object behave like a list. - - :: - - # Call len on results also work - if not results: - print('No match for your sequence') - - # Iterate over results like a list - for match in results: - print(match.encoding, 'can decode properly your sequence using', match.alphabets, 'and language', match.language) - - # Using index to access results - if results: - print(str(results[0])) - -Using best() ------------- - -Like said above, ``CharsetMatches`` object behave like a list and it is sorted by default after getting results from -``from_bytes``, ``from_fp`` or ``from_path``. - -Using ``best()`` return the most probable result, the first entry of the list. Eg. idx 0. -It return a ``CharsetMatch`` object as return value or None if there is not results inside it. - - :: - - result = results.best() - -Calling first() ---------------- - -The very same thing than calling the method ``best()``. - -Class aliases -------------- - -``CharsetMatches`` is also known as ``CharsetDetector``, ``CharsetDoctor`` and ``CharsetNormalizerMatches``. -It is useful if you prefer short class name. - -Verbose output --------------- - -You may want to understand why a specific encoding was not picked by charset_normalizer. All you have to do is passing -``explain`` to True when using methods ``from_bytes``, ``from_fp`` or ``from_path``. diff --git a/tools/retro-library/charset_normalizer/docs/user/cli.rst b/tools/retro-library/charset_normalizer/docs/user/cli.rst deleted file mode 100644 index 6dba8b489..000000000 --- a/tools/retro-library/charset_normalizer/docs/user/cli.rst +++ /dev/null @@ -1,116 +0,0 @@ -Command Line Interface -====================== - -charset-normalizer ship with a CLI that should be available as `normalizer`. -This is a great tool to fully exploit the detector capabilities without having to write Python code. - -Possible use cases: - -#. Quickly discover probable originating charset from a file. -#. I want to quickly convert a non Unicode file to Unicode. -#. Debug the charset-detector. - -Down below, we will guide you through some basic examples. - -Arguments ---------- - -You may simply invoke `normalizer -h` (with the h(elp) flag) to understand the basics. - -:: - - usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD] - file [file ...] - - The Real First Universal Charset Detector. Discover originating encoding used - on text file. Normalize text to unicode. - - positional arguments: - files File(s) to be analysed - - optional arguments: - -h, --help show this help message and exit - -v, --verbose Display complementary information about file if any. - Stdout will contain logs about the detection process. - -a, --with-alternative - Output complementary possibilities if any. Top-level - JSON WILL be a list. - -n, --normalize Permit to normalize input file. If not set, program - does not write anything. - -m, --minimal Only output the charset detected to STDOUT. Disabling - JSON output. - -r, --replace Replace file when trying to normalize it instead of - creating a new one. - -f, --force Replace file without asking if you are sure, use this - flag with caution. - -t THRESHOLD, --threshold THRESHOLD - Define a custom maximum amount of chaos allowed in - decoded content. 0. <= chaos <= 1. - --version Show version information and exit. - -.. code:: bash - - normalizer ./data/sample.1.fr.srt - -You may also run the command line interface using: - -.. code:: bash - - python -m charset_normalizer ./data/sample.1.fr.srt - - -Main JSON Output ----------------- - -🎉 Since version 1.4.0 the CLI produce easily usable stdout result in -JSON format. - -.. code:: json - - { - "path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt", - "encoding": "cp1252", - "encoding_aliases": [ - "1252", - "windows_1252" - ], - "alternative_encodings": [ - "cp1254", - "cp1256", - "cp1258", - "iso8859_14", - "iso8859_15", - "iso8859_16", - "iso8859_3", - "iso8859_9", - "latin_1", - "mbcs" - ], - "language": "French", - "alphabets": [ - "Basic Latin", - "Latin-1 Supplement" - ], - "has_sig_or_bom": false, - "chaos": 0.149, - "coherence": 97.152, - "unicode_path": null, - "is_preferred": true - } - - -I recommend the `jq` command line tool to easily parse and exploit specific data from the produced JSON. - -Multiple File Input -------------------- - -It is possible to give multiple files to the CLI. It will produce a list instead of an object at the top level. -When using the `-m` (minimal output) it will rather print one result (encoding) per line. - -Unicode Conversion ------------------- - -If you desire to convert any file to Unicode you will need to append the flag `-n`. It will produce another file, -it won't replace it by default. - -The newly created file path will be declared in `unicode_path` (JSON output). diff --git a/tools/retro-library/charset_normalizer/docs/user/getstarted.rst b/tools/retro-library/charset_normalizer/docs/user/getstarted.rst deleted file mode 100644 index 47fb2287d..000000000 --- a/tools/retro-library/charset_normalizer/docs/user/getstarted.rst +++ /dev/null @@ -1,73 +0,0 @@ -Installation -============ - -This installs a package that can be used from Python (``import charset_normalizer``). - -To install for all users on the system, administrator rights (root) may be required. - -Using PIP ---------- -Charset Normalizer can be installed from pip:: - - pip install charset-normalizer - -You may retrieve the latest unicodedata backport as follow:: - - pip install charset-normalizer[unicode_backport] - -From git via master ------------------------ -You can install from dev-master branch using git:: - - git clone https://github.com/Ousret/charset_normalizer.git - cd charset_normalizer/ - python setup.py install - -Basic Usage -=========== - -The new way ------------ - -You may want to get right to it. :: - - from charset_normalizer import from_bytes, from_path - - # This is going to print out your sequence once properly decoded - print( - str( - from_bytes( - my_byte_str - ).best() - ) - ) - - # You could also want the same from a file - print( - str( - from_path( - './data/sample.1.ar.srt' - ).best() - ) - ) - - -Backward compatibility ----------------------- - -If you were used to python chardet, we are providing the very same ``detect()`` method as chardet. -This function is mostly backward-compatible with Chardet. The migration should be painless. - - :: - - from charset_normalizer import detect - - # This will behave exactly the same as python chardet - result = detect(my_byte_str) - - if result['encoding'] is not None: - print('got', result['encoding'], 'as detected encoding') - - -You may upgrade your code with ease. -CTRL + R ``from chardet import detect`` to ``from charset_normalizer import detect``. diff --git a/tools/retro-library/charset_normalizer/docs/user/handling_result.rst b/tools/retro-library/charset_normalizer/docs/user/handling_result.rst deleted file mode 100644 index b4a6ac46a..000000000 --- a/tools/retro-library/charset_normalizer/docs/user/handling_result.rst +++ /dev/null @@ -1,25 +0,0 @@ -================ - Handling Result -================ - -When initiating search upon a buffer, bytes or file you can assign the return value and fully exploit it. - - :: - - my_byte_str = 'Bсеки човек има право на образование.'.encode('cp1251') - - # Assign return value so we can fully exploit result - result = from_bytes( - my_byte_str - ).best() - - print(result.encoding) # cp1251 - -Using CharsetMatch ----------------------------- - -Here, ``result`` is a ``CharsetMatch`` object or ``None``. - -.. autoclass:: charset_normalizer.CharsetMatch - :members: - diff --git a/tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst b/tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst deleted file mode 100644 index c6251396d..000000000 --- a/tools/retro-library/charset_normalizer/docs/user/miscellaneous.rst +++ /dev/null @@ -1,64 +0,0 @@ -============== - Miscellaneous -============== - -Convert to str --------------- - -Any ``CharsetMatch`` object can be transformed to exploitable ``str`` variable. - - :: - - my_byte_str = 'Bсеки човек има право на образование.'.encode('cp1251') - - # Assign return value so we can fully exploit result - result = from_bytes( - my_byte_str - ).best() - - # This should print 'Bсеки човек има право на образование.' - print(str(result)) - - -Logging -------- - -Prior to the version 2.0.11 you may encounter some unexpected logs in your streams. -Something along the line of: - - :: - - ... | WARNING | override steps (5) and chunk_size (512) as content does not fit (465 byte(s) given) parameters. - ... | INFO | ascii passed initial chaos probing. Mean measured chaos is 0.000000 % - ... | INFO | ascii should target any language(s) of ['Latin Based'] - - -It is most likely because you altered the root getLogger instance. The package has its own logic behind logging and why -it is useful. See https://docs.python.org/3/howto/logging.html to learn the basics. - -If you are looking to silence and/or reduce drastically the amount of logs, please upgrade to the latest version -available for `charset-normalizer` using your package manager or by `pip install charset-normalizer -U`. - -The latest version will no longer produce any entry greater than `DEBUG`. -On `DEBUG` only one entry will be observed and that is about the detection result. - -Then regarding the others log entries, they will be pushed as `Level 5`. Commonly known as TRACE level, but we do -not register it globally. - - -Detect binaries ---------------- - -This package offers a neat way to detect files that can be considered as 'binaries' -meaning that it is not likely to be a text-file. - - :: - - from charset_normalizer import is_binary - - # It can receive both a path or bytes or even a file pointer. - result = is_binary("./my-file.ext") - - # This should print 'True' or 'False' - print(result) - diff --git a/tools/retro-library/charset_normalizer/docs/user/support.rst b/tools/retro-library/charset_normalizer/docs/user/support.rst deleted file mode 100644 index b00aa9dc2..000000000 --- a/tools/retro-library/charset_normalizer/docs/user/support.rst +++ /dev/null @@ -1,183 +0,0 @@ -================= - Support -================= - -**If you are running:** - -- Python >=2.7,<3.5: Unsupported -- Python 3.5: charset-normalizer < 2.1 -- Python 3.6: charset-normalizer < 3.1 - -Upgrade your Python interpreter as soon as possible. - -------------------- -Supported Encodings -------------------- - -Here are a list of supported encoding and supported language with latest update. Also this list -may change depending of your python version. - -Charset Normalizer is able to detect any of those encoding. This list is NOT static and depends heavily on what your -current cPython version is shipped with. See https://docs.python.org/3/library/codecs.html#standard-encodings - -=============== =============================================================================================================================== -IANA Code Page Aliases -=============== =============================================================================================================================== -ascii 646, ansi_x3.4_1968, ansi_x3_4_1968, ansi_x3.4_1986, cp367, csascii, ibm367, iso646_us, iso_646.irv_1991, iso_ir_6, us, us_ascii -big5 big5_tw, csbig5, x_mac_trad_chinese -big5hkscs big5_hkscs, hkscs -cp037 037, csibm037, ebcdic_cp_ca, ebcdic_cp_nl, ebcdic_cp_us, ebcdic_cp_wt, ibm037, ibm039 -cp1026 1026, csibm1026, ibm1026 -cp1125 1125, ibm1125, cp866u, ruscii -cp1140 1140, ibm1140 -cp1250 1250, windows_1250 -cp1251 1251, windows_1251 -cp1252 1252, windows_1252 -cp1253 1253, windows_1253 -cp1254 1254, windows_1254 -cp1255 1255, windows_1255 -cp1256 1256, windows_1256 -cp1257 1257, windows_1257 -cp1258 1258, windows_1258 -cp273 273, ibm273, csibm273 -cp424 424, csibm424, ebcdic_cp_he, ibm424 -cp437 437, cspc8codepage437, ibm437 -cp500 500, csibm500, ebcdic_cp_be, ebcdic_cp_ch, ibm500 -cp775 775, cspc775baltic, ibm775 -cp850 850, cspc850multilingual, ibm850 -cp852 852, cspcp852, ibm852 -cp855 855, csibm855, ibm855 -cp857 857, csibm857, ibm857 -cp858 858, csibm858, ibm858 -cp860 860, csibm860, ibm860 -cp861 861, cp_is, csibm861, ibm861 -cp862 862, cspc862latinhebrew, ibm862 -cp863 863, csibm863, ibm863 -cp864 864, csibm864, ibm864 -cp865 865, csibm865, ibm865 -cp866 866, csibm866, ibm866 -cp869 869, cp_gr, csibm869, ibm869 -cp932 932, ms932, mskanji, ms_kanji -cp949 949, ms949, uhc -cp950 950, ms950 -euc_jis_2004 jisx0213, eucjis2004, euc_jis2004 -euc_jisx0213 eucjisx0213 -euc_jp eucjp, ujis, u_jis -euc_kr euckr, korean, ksc5601, ks_c_5601, ks_c_5601_1987, ksx1001, ks_x_1001, x_mac_korean -gb18030 gb18030_2000 -gb2312 chinese, csiso58gb231280, euc_cn, euccn, eucgb2312_cn, gb2312_1980, gb2312_80, iso_ir_58, x_mac_simp_chinese -gbk 936, cp936, ms936 -hp_roman8 roman8, r8, csHPRoman8 -hz hzgb, hz_gb, hz_gb_2312 -iso2022_jp csiso2022jp, iso2022jp, iso_2022_jp -iso2022_jp_1 iso2022jp_1, iso_2022_jp_1 -iso2022_jp_2 iso2022jp_2, iso_2022_jp_2 -iso2022_jp_3 iso2022jp_3, iso_2022_jp_3 -iso2022_jp_ext iso2022jp_ext, iso_2022_jp_ext -iso2022_kr csiso2022kr, iso2022kr, iso_2022_kr -iso8859_10 csisolatin6, iso_8859_10, iso_8859_10_1992, iso_ir_157, l6, latin6 -iso8859_11 thai, iso_8859_11, iso_8859_11_2001 -iso8859_13 iso_8859_13, l7, latin7 -iso8859_14 iso_8859_14, iso_8859_14_1998, iso_celtic, iso_ir_199, l8, latin8 -iso8859_15 iso_8859_15, l9, latin9 -iso8859_16 iso_8859_16, iso_8859_16_2001, iso_ir_226, l10, latin10 -iso8859_2 csisolatin2, iso_8859_2, iso_8859_2_1987, iso_ir_101, l2, latin2 -iso8859_3 csisolatin3, iso_8859_3, iso_8859_3_1988, iso_ir_109, l3, latin3 -iso8859_4 csisolatin4, iso_8859_4, iso_8859_4_1988, iso_ir_110, l4, latin4 -iso8859_5 csisolatincyrillic, cyrillic, iso_8859_5, iso_8859_5_1988, iso_ir_144 -iso8859_6 arabic, asmo_708, csisolatinarabic, ecma_114, iso_8859_6, iso_8859_6_1987, iso_ir_127 -iso8859_7 csisolatingreek, ecma_118, elot_928, greek, greek8, iso_8859_7, iso_8859_7_1987, iso_ir_126 -iso8859_8 csisolatinhebrew, hebrew, iso_8859_8, iso_8859_8_1988, iso_ir_138 -iso8859_9 csisolatin5, iso_8859_9, iso_8859_9_1989, iso_ir_148, l5, latin5 -iso2022_jp_2004 iso_2022_jp_2004, iso2022jp_2004 -johab cp1361, ms1361 -koi8_r cskoi8r -kz1048 kz_1048, rk1048, strk1048_2002 -latin_1 8859, cp819, csisolatin1, ibm819, iso8859, iso8859_1, iso_8859_1, iso_8859_1_1987, iso_ir_100, l1, latin, latin1 -mac_cyrillic maccyrillic -mac_greek macgreek -mac_iceland maciceland -mac_latin2 maccentraleurope, maclatin2 -mac_roman macintosh, macroman -mac_turkish macturkish -ptcp154 csptcp154, pt154, cp154, cyrillic_asian -shift_jis csshiftjis, shiftjis, sjis, s_jis, x_mac_japanese -shift_jis_2004 shiftjis2004, sjis_2004, s_jis_2004 -shift_jisx0213 shiftjisx0213, sjisx0213, s_jisx0213 -tis_620 tis620, tis_620_0, tis_620_2529_0, tis_620_2529_1, iso_ir_166 -utf_16 u16, utf16 -utf_16_be unicodebigunmarked, utf_16be -utf_16_le unicodelittleunmarked, utf_16le -utf_32 u32, utf32 -utf_32_be utf_32be -utf_32_le utf_32le -utf_8 u8, utf, utf8, utf8_ucs2, utf8_ucs4 (+utf_8_sig) -utf_7* u7, unicode-1-1-utf-7 -cp720 N.A. -cp737 N.A. -cp856 N.A. -cp874 N.A. -cp875 N.A. -cp1006 N.A. -koi8_r N.A. -koi8_t N.A. -koi8_u N.A. -=============== =============================================================================================================================== - -*: Only if a SIG/mark is found. - -------------------- -Supported Languages -------------------- - -Those language can be detected inside your content. All of these are specified in ./charset_normalizer/assets/__init__.py . - - -| English, -| German, -| French, -| Dutch, -| Italian, -| Polish, -| Spanish, -| Russian, -| Japanese, -| Portuguese, -| Swedish, -| Chinese, -| Ukrainian, -| Norwegian, -| Finnish, -| Vietnamese, -| Czech, -| Hungarian, -| Korean, -| Indonesian, -| Turkish, -| Romanian, -| Farsi, -| Arabic, -| Danish, -| Serbian, -| Lithuanian, -| Slovene, -| Slovak, -| Malay, -| Hebrew, -| Bulgarian, -| Croatian, -| Hindi, -| Estonian, -| Thai, -| Greek, -| Tamil. - ----------------------------- -Incomplete Sequence / Stream ----------------------------- - -It is not (yet) officially supported. If you feed an incomplete byte sequence (eg. truncated multi-byte sequence) the detector will -most likely fail to return a proper result. -If you are purposely feeding part of your payload for performance concerns, you may stop doing it as this package is fairly optimized. - -We are working on a dedicated way to handle streams. diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/legacy.py b/tools/retro-library/charset_normalizer/legacy.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/legacy.py rename to tools/retro-library/charset_normalizer/legacy.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/md.py b/tools/retro-library/charset_normalizer/md.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/md.py rename to tools/retro-library/charset_normalizer/md.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/models.py b/tools/retro-library/charset_normalizer/models.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/models.py rename to tools/retro-library/charset_normalizer/models.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/py.typed b/tools/retro-library/charset_normalizer/py.typed similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/py.typed rename to tools/retro-library/charset_normalizer/py.typed diff --git a/tools/retro-library/charset_normalizer/pyproject.toml b/tools/retro-library/charset_normalizer/pyproject.toml deleted file mode 100644 index 05f32e5ac..000000000 --- a/tools/retro-library/charset_normalizer/pyproject.toml +++ /dev/null @@ -1,81 +0,0 @@ -[build-system] -requires = ["setuptools", "setuptools-scm", "mypy>=1.4.1,<=1.13.0"] -build-backend = "setuptools.build_meta" - -[project] -name = "charset-normalizer" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -license = {text = "MIT"} -keywords = ["encoding", "charset", "charset-detector", "detector", "normalization", "unicode", "chardet", "detect"] -authors = [ - {name = "Ahmed R. TAHRI", email="tahri.ahmed@proton.me"}, -] -maintainers = [ - {name = "Ahmed R. TAHRI", email="tahri.ahmed@proton.me"}, -] -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Text Processing :: Linguistic", - "Topic :: Utilities", - "Typing :: Typed", -] -requires-python = ">=3.7" -dynamic = ["version", "readme"] - -[project.optional-dependencies] -unicode_backport = [] - -[tool.setuptools.dynamic] -version = {attr = "charset_normalizer.__version__"} -readme = {file = ["README.md", "CHANGELOG.md", "LICENSE"]} - -[project.scripts] -normalizer = "charset_normalizer:cli.cli_detect" - -[project.urls] -"Changelog" = "https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md" -"Documentation" = "https://charset-normalizer.readthedocs.io/" -"Code" = "https://github.com/jawah/charset_normalizer" -"Issue tracker" = "https://github.com/jawah/charset_normalizer/issues" - -[tool.setuptools.packages.find] -exclude = ["tests*"] - -[tool.pytest.ini_options] -addopts = "--cov=charset_normalizer --cov-report=term-missing -rxXs" - -[tool.isort] -profile = "black" -add_imports = "from __future__ import annotations" - -[tool.mypy] -check_untyped_defs = true -disallow_any_generics = true -disallow_incomplete_defs = true -disallow_subclassing_any = true -disallow_untyped_calls = true -disallow_untyped_decorators = true -disallow_untyped_defs = true -no_implicit_optional = true -no_implicit_reexport = true -show_error_codes = true -strict_equality = true -warn_redundant_casts = true -warn_return_any = true -warn_unused_configs = true -warn_unused_ignores = false diff --git a/tools/retro-library/charset_normalizer/setup.cfg b/tools/retro-library/charset_normalizer/setup.cfg deleted file mode 100644 index 5d23e2539..000000000 --- a/tools/retro-library/charset_normalizer/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -ignore = W503, E203, B305 -max-line-length = 120 diff --git a/tools/retro-library/charset_normalizer/setup.py b/tools/retro-library/charset_normalizer/setup.py deleted file mode 100644 index c113acda9..000000000 --- a/tools/retro-library/charset_normalizer/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -from __future__ import annotations - -import os -import sys - -from setuptools import setup - -USE_MYPYC = False - -if len(sys.argv) > 1 and sys.argv[1] == "--use-mypyc": - sys.argv.pop(1) - USE_MYPYC = True -if os.getenv("CHARSET_NORMALIZER_USE_MYPYC", None) == "1": - USE_MYPYC = True - -if USE_MYPYC: - from mypyc.build import mypycify - - MYPYC_MODULES = mypycify( - [ - "charset_normalizer/md.py", - ], - debug_level="0", - ) -else: - MYPYC_MODULES = None - -setup(name="charset-normalizer", ext_modules=MYPYC_MODULES) diff --git a/tools/retro-library/charset_normalizer/tests/__init__.py b/tools/retro-library/charset_normalizer/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tools/retro-library/charset_normalizer/tests/test_base_detection.py b/tools/retro-library/charset_normalizer/tests/test_base_detection.py deleted file mode 100644 index e4fb5fd39..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_base_detection.py +++ /dev/null @@ -1,162 +0,0 @@ -from __future__ import annotations - -import pytest - -from charset_normalizer.api import from_bytes -from charset_normalizer.models import CharsetMatches - - -def test_empty(): - best_guess = from_bytes(b"").best() - - assert best_guess is not None, "Empty bytes payload SHOULD NOT return None" - assert ( - best_guess.encoding == "utf_8" - ), "Empty bytes payload SHOULD be guessed as UTF-8 (arbitrary)" - assert len(best_guess.alphabets) == 0, "" - - -def test_bool_matches(): - guesses_not_empty = from_bytes(b"") - guesses_empty = CharsetMatches([]) - - assert ( - bool(guesses_not_empty) is True - ), "Bool behaviour of CharsetMatches altered, should be True" - assert ( - bool(guesses_empty) is False - ), "Bool behaviour of CharsetMatches altered, should be False" - - -@pytest.mark.parametrize( - "payload, expected_encoding", - [ - (b"\xfe\xff", "utf_16"), - ("\uFEFF".encode("gb18030"), "gb18030"), - (b"\xef\xbb\xbf", "utf_8"), - ("".encode("utf_32"), "utf_32"), - ], -) -def test_empty_but_with_bom_or_sig(payload, expected_encoding): - best_guess = from_bytes(payload).best() - - assert best_guess is not None, "Empty detection but with SIG/BOM has failed!" - assert ( - best_guess.encoding == expected_encoding - ), "Empty detection but with SIG/BOM is wrongly detected!" - assert ( - best_guess.raw == payload - ), "The RAW property should contain the original payload given for detection." - assert best_guess.byte_order_mark is True, "The BOM/SIG property should return True" - assert str(best_guess) == "", "The cast to str SHOULD be empty" - - -@pytest.mark.parametrize( - "payload, expected_encoding", - [ - ( - ("\uFEFF" + "我没有埋怨，磋砣的只是一些时间。").encode("gb18030"), - "gb18030", - ), - ( - "我没有埋怨，磋砣的只是一些时间。".encode("utf_32"), - "utf_32", - ), - ( - "我没有埋怨，磋砣的只是一些时间。".encode("utf_8_sig"), - "utf_8", - ), - ], -) -def test_content_with_bom_or_sig(payload, expected_encoding): - best_guess = from_bytes(payload).best() - - assert best_guess is not None, "Detection but with SIG/BOM has failed!" - assert ( - best_guess.encoding == expected_encoding - ), "Detection but with SIG/BOM is wrongly detected!" - assert best_guess.byte_order_mark is True, "The BOM/SIG property should return True" - - -@pytest.mark.parametrize( - "payload", - [ - b"AbAdZ pOoooOlDl mmlDoDkA lldDkeEkddA mpAlkDF", - b"g4UsPJdfzNkGW2jwmKDGDilKGKYtpF2X.mx3MaTWL1tL7CNn5U7DeCcodKX7S3lwwJPKNjBT8etY", - b'{"token": "g4UsPJdfzNkGW2jwmKDGDilKGKYtpF2X.mx3MaTWL1tL7CNn5U7DeCcodKX7S3lwwJPKNjBT8etY"}', - b"81f4ab054b39cb0e12701e734077d84264308f5fc79494fc5f159fa2ebc07b73c8cc0e98e009664a20986706f90146e8eefcb929ce1f74a8eab21369fdc70198", - b"{}", - ], -) -def test_obviously_ascii_content(payload): - best_guess = from_bytes(payload).best() - - assert best_guess is not None, "Dead-simple ASCII detection has failed!" - assert ( - best_guess.encoding == "ascii" - ), "Dead-simple ASCII detection is wrongly detected!" - - -@pytest.mark.parametrize( - "payload", - [ - "\u020d\x1b".encode(), - "h\xe9llo world!\n".encode(), - "我没有埋怨，磋砣的只是一些时间。".encode(), - "Bсеки човек има право на образование. Oбразованието трябва да бъде безплатно, поне що се отнася до началното и основното образование.".encode(), - "Bсеки човек има право на образование.".encode(), - "(° ͜ʖ °), creepy face, smiley 😀".encode(), - """["Financiën", "La France"]""".encode(), - "Qu'est ce que une étoile?".encode(), - """Financiën""".encode(), - "😀".encode(), - ], -) -def test_obviously_utf8_content(payload): - best_guess = from_bytes(payload).best() - - assert best_guess is not None, "Dead-simple UTF-8 detection has failed!" - assert ( - best_guess.encoding == "utf_8" - ), "Dead-simple UTF-8 detection is wrongly detected!" - - -def test_mb_cutting_chk(): - # This payload should be wrongfully split and the autofix should ran automatically - # on chunks extraction. - payload = ( - b"\xbf\xaa\xbb\xe7\xc0\xfb \xbf\xb9\xbc\xf6 " - b" \xbf\xac\xb1\xb8\xc0\xda\xb5\xe9\xc0\xba \xba\xb9\xc0\xbd\xbc\xad\xb3\xaa " - * 128 - ) - - guesses = from_bytes(payload, cp_isolation=["cp949"]) - best_guess = guesses.best() - - assert len(guesses) == 1, "cp isolation is set and given seq should be clear CP949!" - assert best_guess.encoding == "cp949" - - -def test_alphabets_property(): - best_guess = from_bytes("😀 Hello World! How affairs are going? 😀".encode()).best() - - assert "Basic Latin" in best_guess.alphabets - assert "Emoticons range(Emoji)" in best_guess.alphabets - assert best_guess.alphabets.count("Basic Latin") == 1 - - -def test_doc_example_short_cp1251(): - best_guess = from_bytes( - "Bсеки човек има право на образование.".encode("cp1251") - ).best() - - assert best_guess.encoding == "cp1251" - - -def test_direct_cmp_charset_match(): - best_guess = from_bytes("😀 Hello World! How affairs are going? 😀".encode()).best() - - assert best_guess == "utf_8" - assert best_guess == "utf-8" - assert best_guess != 8 - assert best_guess != None diff --git a/tools/retro-library/charset_normalizer/tests/test_cli.py b/tools/retro-library/charset_normalizer/tests/test_cli.py deleted file mode 100644 index 5f2777c41..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_cli.py +++ /dev/null @@ -1,116 +0,0 @@ -from __future__ import annotations - -import unittest -from os import pardir, path, remove -from os.path import exists -from unittest.mock import patch - -from charset_normalizer.cli import cli_detect, query_yes_no - -DIR_PATH = path.join(path.dirname(path.realpath(__file__)), pardir) - - -class TestCommandLineInterface(unittest.TestCase): - @patch("builtins.input", lambda *args: "y") - def test_simple_yes_input(self): - self.assertTrue(query_yes_no("Are u willing to chill a little bit ?")) - - @patch("builtins.input", lambda *args: "N") - def test_simple_no_input(self): - self.assertFalse(query_yes_no("Are u willing to chill a little bit ?")) - - def test_single_file(self): - self.assertEqual(0, cli_detect([DIR_PATH + "/data/sample-arabic-1.txt"])) - - def test_version_output_success(self): - with self.assertRaises(SystemExit): - cli_detect(["--version"]) - - def test_single_file_normalize(self): - self.assertEqual( - 0, cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--normalize"]) - ) - - self.assertTrue(exists(DIR_PATH + "/data/sample-arabic-1.cp1256.txt")) - - try: - remove(DIR_PATH + "/data/sample-arabic-1.cp1256.txt") - except: - pass - - def test_single_verbose_file(self): - self.assertEqual( - 0, cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--verbose"]) - ) - - def test_multiple_file(self): - self.assertEqual( - 0, - cli_detect( - [ - DIR_PATH + "/data/sample-arabic-1.txt", - DIR_PATH + "/data/sample-french.txt", - DIR_PATH + "/data/sample-chinese.txt", - ] - ), - ) - - def test_with_alternative(self): - self.assertEqual( - 0, - cli_detect( - [ - "-a", - DIR_PATH + "/data/sample-arabic-1.txt", - DIR_PATH + "/data/sample-french.txt", - DIR_PATH + "/data/sample-chinese.txt", - ] - ), - ) - - def test_with_minimal_output(self): - self.assertEqual( - 0, - cli_detect( - [ - "-m", - DIR_PATH + "/data/sample-arabic-1.txt", - DIR_PATH + "/data/sample-french.txt", - DIR_PATH + "/data/sample-chinese.txt", - ] - ), - ) - - def test_with_minimal_and_alt(self): - self.assertEqual( - 0, - cli_detect( - [ - "-m", - "-a", - DIR_PATH + "/data/sample-arabic-1.txt", - DIR_PATH + "/data/sample-french.txt", - DIR_PATH + "/data/sample-chinese.txt", - ] - ), - ) - - def test_non_existent_file(self): - with self.assertRaises(SystemExit) as cm: - cli_detect([DIR_PATH + "/data/not_found_data.txt"]) - - self.assertEqual(cm.exception.code, 2) - - def test_replace_without_normalize(self): - self.assertEqual( - cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--replace"]), 1 - ) - - def test_force_replace_without_replace(self): - self.assertEqual( - cli_detect([DIR_PATH + "/data/sample-arabic-1.txt", "--force"]), 1 - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/retro-library/charset_normalizer/tests/test_coherence_detection.py b/tools/retro-library/charset_normalizer/tests/test_coherence_detection.py deleted file mode 100644 index e5952d6c0..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_coherence_detection.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -import pytest - -from charset_normalizer.cd import ( - encoding_languages, - filter_alt_coherence_matches, - get_target_features, - is_multi_byte_encoding, - mb_encoding_languages, -) - - -@pytest.mark.parametrize( - "iana_encoding, expected_languages", - [ - ("cp864", ["Arabic", "Farsi"]), - ("cp862", ["Hebrew"]), - ("cp737", ["Greek"]), - ("cp424", ["Hebrew"]), - ("cp273", ["Latin Based"]), - ("johab", ["Korean"]), - ("shift_jis", ["Japanese"]), - ("mac_greek", ["Greek"]), - ("iso2022_jp", ["Japanese"]), - ], -) -def test_infer_language_from_cp(iana_encoding, expected_languages): - languages = ( - mb_encoding_languages(iana_encoding) - if is_multi_byte_encoding(iana_encoding) - else encoding_languages(iana_encoding) - ) - - for expected_language in expected_languages: - assert ( - expected_language in languages - ), "Wrongly detected language for given code page" - - -@pytest.mark.parametrize( - "language, expected_have_accents, expected_pure_latin", - [ - ("English", False, True), - ("French", True, True), - ("Hebrew", False, False), - ("Arabic", False, False), - ("Vietnamese", True, True), - ("Turkish", True, True), - ], -) -def test_target_features(language, expected_have_accents, expected_pure_latin): - target_have_accents, target_pure_latin = get_target_features(language) - - assert target_have_accents is expected_have_accents - assert target_pure_latin is expected_pure_latin - - -@pytest.mark.parametrize( - "matches, expected_return", - [ - ( - [ - ( - "English", - 0.88, - ), - ("English—", 0.99), - ], - [("English", 0.99)], - ), - ( - [ - ( - "English", - 0.88, - ), - ("English—", 0.99), - ("English——", 0.999), - ], - [("English", 0.999)], - ), - ( - [ - ( - "English", - 0.88, - ), - ("English—", 0.77), - ], - [("English", 0.88)], - ), - ( - [ - ( - "English", - 0.88, - ), - ("Italian", 0.77), - ], - [("English", 0.88), ("Italian", 0.77)], - ), - ], -) -def test_filter_alt_coherence_matches(matches, expected_return): - results = filter_alt_coherence_matches(matches) - - assert results == expected_return diff --git a/tools/retro-library/charset_normalizer/tests/test_detect_legacy.py b/tools/retro-library/charset_normalizer/tests/test_detect_legacy.py deleted file mode 100644 index bd2b03517..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_detect_legacy.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -import unittest - -from charset_normalizer.legacy import detect - - -class TestDetectLegacy(unittest.TestCase): - def test_detect_dict_keys(self): - r = detect(("\uFEFF" + "我没有埋怨，磋砣的只是一些时间。").encode("gb18030")) - - with self.subTest("encoding key present"): - self.assertIn("encoding", r.keys()) - - with self.subTest("language key present"): - self.assertIn("language", r.keys()) - - with self.subTest("confidence key present"): - self.assertIn("confidence", r.keys()) - - def test_detect_dict_value_type(self): - r = detect("我没有埋怨，磋砣的只是一些时间。".encode()) - - with self.subTest("encoding instance of str"): - self.assertIsInstance(r["encoding"], str) - - with self.subTest("language instance of str"): - self.assertIsInstance(r["language"], str) - - with self.subTest("confidence instance of float"): - self.assertIsInstance(r["confidence"], float) - - def test_detect_dict_value(self): - r = detect("我没有埋怨，磋砣的只是一些时间。".encode("utf_32")) - - with self.subTest("encoding is equal to utf_32"): - self.assertEqual(r["encoding"], "UTF-32") - - def test_utf8_sig_not_striped(self): - r = detect("Hello World".encode("utf-8-sig")) - - with self.subTest("Verify that UTF-8-SIG is returned when using legacy detect"): - self.assertEqual(r["encoding"], "UTF-8-SIG") diff --git a/tools/retro-library/charset_normalizer/tests/test_edge_case.py b/tools/retro-library/charset_normalizer/tests/test_edge_case.py deleted file mode 100644 index 5b763ba27..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_edge_case.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import annotations - -import platform - -import pytest - -from charset_normalizer import from_bytes - - -@pytest.mark.xfail( - platform.python_version_tuple()[0] == "3" - and platform.python_version_tuple()[1] == "7", - reason="Unicode database is too old for this case (Python 3.7)", -) -def test_unicode_edge_case(): - payload = b"\xef\xbb\xbf\xf0\x9f\xa9\xb3" - - best_guess = from_bytes(payload).best() - - assert ( - best_guess is not None - ), "Payload should have given something, detection failure" - assert best_guess.encoding == "utf_8", "UTF-8 payload wrongly detected" - - -def test_issue_gh520(): - """Verify that minorities does not strip basic latin characters!""" - payload = b"/includes/webform.compon\xd2\xaants.inc/" - - best_guess = from_bytes(payload).best() - - assert ( - best_guess is not None - ), "Payload should have given something, detection failure" - assert "Basic Latin" in best_guess.alphabets - - -def test_issue_gh509(): - """Two common ASCII punctuations should render as-is.""" - payload = b");" - - best_guess = from_bytes(payload).best() - - assert ( - best_guess is not None - ), "Payload should have given something, detection failure" - assert "ascii" == best_guess.encoding - - -def test_issue_gh498(): - """This case was mistaken for utf-16-le, this should never happen again.""" - payload = b"\x84\xae\xaa\xe3\xac\xa5\xad\xe2 Microsoft Word.docx" - - best_guess = from_bytes(payload).best() - - assert ( - best_guess is not None - ), "Payload should have given something, detection failure" - assert "Cyrillic" in best_guess.alphabets diff --git a/tools/retro-library/charset_normalizer/tests/test_full_detection.py b/tools/retro-library/charset_normalizer/tests/test_full_detection.py deleted file mode 100644 index ff91e1250..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_full_detection.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -from os import pardir, path - -import pytest - -from charset_normalizer.api import from_path - -DIR_PATH = path.join(path.dirname(path.realpath(__file__)), pardir) - - -@pytest.mark.parametrize( - "input_data_file, expected_charset, expected_language", - [ - ("sample-arabic-1.txt", "cp1256", "Arabic"), - ("sample-french-1.txt", "cp1252", "French"), - ("sample-arabic.txt", "utf_8", "Arabic"), - ("sample-russian-3.txt", "utf_8", "Russian"), - ("sample-french.txt", "utf_8", "French"), - ("sample-chinese.txt", "big5", "Chinese"), - ("sample-greek.txt", "cp1253", "Greek"), - ("sample-greek-2.txt", "cp1253", "Greek"), - ("sample-hebrew-2.txt", "cp1255", "Hebrew"), - ("sample-hebrew-3.txt", "cp1255", "Hebrew"), - ("sample-bulgarian.txt", "utf_8", "Bulgarian"), - ("sample-english.bom.txt", "utf_8", "English"), - ("sample-spanish.txt", "utf_8", "Spanish"), - ("sample-korean.txt", "cp949", "Korean"), - ("sample-turkish.txt", "cp1254", "Turkish"), - ("sample-russian-2.txt", "utf_8", "Russian"), - ("sample-russian.txt", "mac_cyrillic", "Russian"), - ("sample-polish.txt", "utf_8", "Polish"), - ], -) -def test_elementary_detection( - input_data_file: str, - expected_charset: str, - expected_language: str, -): - best_guess = from_path(DIR_PATH + f"/data/{input_data_file}").best() - - assert ( - best_guess is not None - ), f"Elementary detection has failed upon '{input_data_file}'" - assert ( - best_guess.encoding == expected_charset - ), f"Elementary charset detection has failed upon '{input_data_file}'" - assert ( - best_guess.language == expected_language - ), f"Elementary language detection has failed upon '{input_data_file}'" diff --git a/tools/retro-library/charset_normalizer/tests/test_isbinary.py b/tools/retro-library/charset_normalizer/tests/test_isbinary.py deleted file mode 100644 index 841474f10..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_isbinary.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -import typing -from base64 import b64decode -from io import BytesIO -from os import pardir, path - -import pytest - -from charset_normalizer import is_binary - -DIR_PATH = path.join(path.dirname(path.realpath(__file__)), pardir) - - -@pytest.mark.parametrize( - "raw, expected", - [ - (b"\x00\x5f\x2f\xff" * 50, True), - (b64decode("R0lGODlhAQABAAAAACw="), True), - (BytesIO(b64decode("R0lGODlhAQABAAAAACw=")), True), - ("sample-polish.txt", False), - ("sample-arabic.txt", False), - ], -) -def test_isbinary(raw: bytes | typing.BinaryIO | str, expected: bool) -> None: - if isinstance(raw, str): - raw = DIR_PATH + f"/data/{raw}" - - assert is_binary(raw) is expected diff --git a/tools/retro-library/charset_normalizer/tests/test_large_payload.py b/tools/retro-library/charset_normalizer/tests/test_large_payload.py deleted file mode 100644 index 7fc28fac1..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_large_payload.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -import pytest - -from charset_normalizer import from_bytes -from charset_normalizer.constant import TOO_BIG_SEQUENCE - - -def test_large_payload_u8_sig_basic_entry(): - payload = ("0" * TOO_BIG_SEQUENCE).encode("utf_8_sig") - best_guess = from_bytes(payload).best() - - assert best_guess is not None, "Large U8 payload case detection completely failed" - assert ( - best_guess.encoding == "utf_8" - ), "Large U8 payload case detection wrongly detected!" - assert best_guess.bom is True, "SIG/BOM property should be True" - assert len(best_guess.raw) == len( - payload - ), "Large payload should remain untouched when accessed through .raw" - assert ( - best_guess._string is not None - ), "str should be decoded before direct access (sig available)" - - -def test_large_payload_ascii_basic_entry(): - payload = ("0" * TOO_BIG_SEQUENCE).encode("utf_8") - best_guess = from_bytes(payload).best() - - assert ( - best_guess is not None - ), "Large ASCII payload case detection completely failed" - assert ( - best_guess.encoding == "ascii" - ), "Large ASCII payload case detection wrongly detected!" - assert best_guess.bom is False, "SIG/BOM property should be False" - assert len(best_guess.raw) == len( - payload - ), "Large payload should remain untouched when accessed through .raw" - assert best_guess._string is None, "str should not be decoded until direct access" - - -def test_misleading_large_sequence(): - content = ( - ("hello simple ascii " * TOO_BIG_SEQUENCE) + ("我没有埋怨，磋砣的只是一些时间。 磋砣的只是一些时间。") - ).encode("utf_8") - - guesses = from_bytes(content) - - assert len(guesses) > 0 - match = guesses.best() - assert match is not None - assert match._string is not None, "str should be cached as only match" - assert match.encoding == "utf_8" - assert str(match) is not None diff --git a/tools/retro-library/charset_normalizer/tests/test_logging.py b/tools/retro-library/charset_normalizer/tests/test_logging.py deleted file mode 100644 index ad2413e24..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_logging.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -import logging - -import pytest - -from charset_normalizer.api import explain_handler, from_bytes -from charset_normalizer.constant import TRACE -from charset_normalizer.utils import set_logging_handler - - -class TestLogBehaviorClass: - def setup_method(self): - self.logger = logging.getLogger("charset_normalizer") - self.logger.handlers.clear() - self.logger.addHandler(logging.NullHandler()) - self.logger.level = logging.WARNING - - def test_explain_true_behavior(self, caplog): - test_sequence = b"This is a test sequence of bytes that should be sufficient" - from_bytes(test_sequence, steps=1, chunk_size=50, explain=True) - assert explain_handler not in self.logger.handlers - for record in caplog.records: - assert record.levelname in ["Level 5", "DEBUG"] - - def test_explain_false_handler_set_behavior(self, caplog): - test_sequence = b"This is a test sequence of bytes that should be sufficient" - set_logging_handler(level=TRACE, format_string="%(message)s") - from_bytes(test_sequence, steps=1, chunk_size=50, explain=False) - assert any( - isinstance(hdl, logging.StreamHandler) for hdl in self.logger.handlers - ) - for record in caplog.records: - assert record.levelname in ["Level 5", "DEBUG"] - assert "Encoding detection: ascii is most likely the one." in caplog.text - - def test_set_stream_handler(self, caplog): - set_logging_handler("charset_normalizer", level=logging.DEBUG) - self.logger.debug("log content should log with default format") - for record in caplog.records: - assert record.levelname in ["Level 5", "DEBUG"] - assert "log content should log with default format" in caplog.text - - def test_set_stream_handler_format(self, caplog): - set_logging_handler("charset_normalizer", format_string="%(message)s") - self.logger.info("log content should only be this message") - assert caplog.record_tuples == [ - ( - "charset_normalizer", - logging.INFO, - "log content should only be this message", - ) - ] diff --git a/tools/retro-library/charset_normalizer/tests/test_mess_detection.py b/tools/retro-library/charset_normalizer/tests/test_mess_detection.py deleted file mode 100644 index 4089f8259..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_mess_detection.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -import pytest - -from charset_normalizer.md import mess_ratio - - -@pytest.mark.parametrize( - "content, min_expected_ratio, max_expected_ratio", - [ - ( - "典肇乎庚辰年十二月廿一，及己丑年二月十九，收各方語言二百五十，合逾七百萬目；二十大卷佔八成，單英文卷亦過二百萬。悉文乃天下有志共筆而成；有意助之，幾網路、隨纂作，大典茁焉。", - 0.0, - 0.0, - ), - ("العقلية , التنويم المغناطيسي و / أو الاقتراح", 0.0, 0.0), - ("RadoZ تـــعــــديــل الـــتــــوقــيــــت مـــن قــبــل", 0.0, 0.0), - ("Cehennemin Sava■þ²s²'da kim?", 0.1, 0.5), - ("´Á¥½³ø§i -- ±i®Ìºû, ³¯·Ø©v", 0.5, 1.0), - ( - "ïstanbul, T■rkiye'nin en kalabal»k, iktisadi ve k■lt■rel aÓ»dan en —nemli", - 0.1, - 0.5, - ), - ( - "Parce que Óa, c'est la vÕritable histoire de la rencontre avec votre Tante Robin.", - 0.01, - 0.5, - ), - ( - """ØĢØŠØģاØĶŲ„ Ų„Ųˆ ØĢŲ† اŲ„Ų†Ø§Øģ ŲŠŲˆŲ… Ų…ا ØģŲˆŲŲŠØŠØģاØĶŲ„ŲˆŲ†ØŒ ØŊØđŲ†Ø§ Ų†ØģŲ…Øđ ØđŲ† (ŲØąŲˆØŊŲˆ) ŲˆØ§Ų„ØŪا؊Ų…""", - 0.8, - 3.0, - ), - ("""ÇáÚŞáíÉ , ÇáÊäæíã ÇáãÛäÇØíÓí æ / Ãæ ÇáÇŞÊÑÇÍ""", 0.8, 2.5), - ( - """hishamkoc@yahoo.com ุชุฑุฌู…ู€ู€ุฉ ู‡ู€ุดู€ู€ู€ุงู… ุงู„ู€ู‚ู€ู€ู€ู€ู„ุงูRadoZ ุชู€ู€ู€ุนู€ู€ู€ู€ุฏูŠู€ู€ู„ ุงู„ู€ู€ู€ุชู€ู€ู€ู€ูˆู‚ู€ู€ูŠู€ู€ู€ู€ุช ู…ู€ู€ู€ู† ู‚ู€ู€ุจู€ู€ู„""", - 0.5, - 2.0, - ), - ], -) -def test_mess_detection(content, min_expected_ratio, max_expected_ratio): - calculated_mess_ratio = mess_ratio(content, maximum_threshold=1.0) - - assert ( - min_expected_ratio <= calculated_mess_ratio <= max_expected_ratio - ), "The mess detection ratio calculated for given content is not well adjusted!" diff --git a/tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py b/tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py deleted file mode 100644 index 64b520235..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_preemptive_detection.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import annotations - -import pytest - -from charset_normalizer import CharsetMatch -from charset_normalizer.utils import any_specified_encoding - - -@pytest.mark.parametrize( - "payload, expected_encoding", - [ - (b'', "euc_jp"), - (b'', "utf_8"), - (b'', None), - (b"# coding: utf-8", "utf_8"), - (b'', "utf_8"), - (b'', "ascii"), - (b'', "johab"), - (b'', "cp037"), - (b"", "cp1252"), - (b'', "cp1256"), - ], -) -def test_detect_most_common_body_encoding(payload, expected_encoding): - specified_encoding = any_specified_encoding(payload) - - assert ( - specified_encoding == expected_encoding - ), "Unable to determine properly encoding from given body" - - -@pytest.mark.parametrize( - "payload, expected_outcome", - [ - ( - b'', - b'', - ), - ( - b'', - b'', - ), - ( - b'', - b'', - ), - (b"# coding: utf-8", b"# coding: utf-8"), - ( - b'', - b'', - ), - ( - b'', - b'', - ), - ( - b'', - b'', - ), - ( - b"", - b"", - ), - ( - b'', - b'', - ), - ], -) -def test_preemptive_mark_replacement(payload, expected_outcome): - """ - When generating (to Unicode converted) bytes, we want to change any potential declarative charset - to utf-8. This test that. - """ - specified_encoding = any_specified_encoding(payload) - - detected_encoding = ( - specified_encoding if specified_encoding is not None else "utf-8" - ) - - m = CharsetMatch( - payload, - detected_encoding, - 0.0, - False, - [], - preemptive_declaration=specified_encoding, - ) - - transformed_output = m.output() - - assert transformed_output == expected_outcome diff --git a/tools/retro-library/charset_normalizer/tests/test_utils.py b/tools/retro-library/charset_normalizer/tests/test_utils.py deleted file mode 100644 index a0cc088ed..000000000 --- a/tools/retro-library/charset_normalizer/tests/test_utils.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import logging - -import pytest - -from charset_normalizer.utils import cp_similarity, is_accentuated, set_logging_handler - - -@pytest.mark.parametrize( - "character, expected_is_accentuated", - [ - ("é", True), - ("è", True), - ("à", True), - ("À", True), - ("Ù", True), - ("ç", True), - ("a", False), - ("€", False), - ("&", False), - ("Ö", True), - ("ü", True), - ("ê", True), - ("Ñ", True), - ("Ý", True), - ("Ω", False), - ("ø", False), - ("Ё", False), - ], -) -def test_is_accentuated(character, expected_is_accentuated): - assert ( - is_accentuated(character) is expected_is_accentuated - ), "is_accentuated behavior incomplete" - - -@pytest.mark.parametrize( - "cp_name_a, cp_name_b, expected_is_similar", - [ - ("cp1026", "cp1140", True), - ("cp1140", "cp1026", True), - ("latin_1", "cp1252", True), - ("latin_1", "iso8859_4", True), - ("latin_1", "cp1251", False), - ("cp1251", "mac_turkish", False), - ], -) -def test_cp_similarity(cp_name_a, cp_name_b, expected_is_similar): - is_similar = cp_similarity(cp_name_a, cp_name_b) >= 0.8 - - assert is_similar is expected_is_similar, "cp_similarity is broken" diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/utils.py b/tools/retro-library/charset_normalizer/utils.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/utils.py rename to tools/retro-library/charset_normalizer/utils.py diff --git a/tools/retro-library/charset_normalizer/charset_normalizer/version.py b/tools/retro-library/charset_normalizer/version.py similarity index 100% rename from tools/retro-library/charset_normalizer/charset_normalizer/version.py rename to tools/retro-library/charset_normalizer/version.py From cdfdcba0416bd7ad24c84377324da695e0ea0c6c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 19 Nov 2024 21:13:55 +0100 Subject: [PATCH 082/230] json.dumps --- tools/retro-library/retro_achievements.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py index ee5a89f1c..700852dff 100644 --- a/tools/retro-library/retro_achievements.py +++ b/tools/retro-library/retro_achievements.py @@ -1,6 +1,7 @@ import requests import sys import warnings +import json from requests.exceptions import RequestsDependencyWarning warnings.filterwarnings("ignore", category=RequestsDependencyWarning) @@ -64,7 +65,8 @@ def main(): # Obtener datos del juego y progreso del usuario game_data = get_game_info_and_progress(game_id) if game_data: - print(game_data) + # Imprimir JSON en formato válido + print(json.dumps(game_data, ensure_ascii=False, indent=4)) else: print("Hash MD5 no encontrado en la lista de juegos.") From 40b323447ab079ede1d77d88f47e63248cfcd88f Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 00:58:22 +0100 Subject: [PATCH 083/230] new paths --- functions/generateGameLists.sh | 23 +++++++++++++--------- tools/retro-library/generate_game_lists.py | 2 +- tools/retro-library/missing_artwork.py | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 22bd1f962..72ea034c3 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -1,8 +1,15 @@ #!/bin/bash generateGameLists() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/emudeck/" - mkdir -p $dest_folder + local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" + + mkdir -p "$storagePath/retrolibrary/artwork" + mkdir -p "$storagePath/retrolibrary/data" + mkdir -p "$accountfolder/config/grid/retrolibrary" + + ln -s "$storagePath/retrolibrary/artwork" "$accountfolder/config/grid/retrolibrary/artwork" + ln -s "$storagePath/retrolibrary/data" "$accountfolder/config/grid/retrolibrary/data" + pegasus_setPaths rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$dest_folder" --keep-dirlinks mkdir -p "$HOME/emudeck/cache/" @@ -18,15 +25,13 @@ generateGameListsJson() { } generateGameLists_importESDE() { - - python $HOME/.config/EmuDeck/backend/tools/retro-library/import_media.py "$romsPath" "$dest_folder" } generateGameLists_artwork() { mkdir -p "$HOME/emudeck/cache/" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/emudeck/" + local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" @@ -37,7 +42,7 @@ saveImage(){ local name=$2 local system=$3 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/emudeck/${system}" + local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/${system}" local dest_path="$dest_folder/$name.jpg" wget -q -O "$dest_path" "$url" } @@ -52,7 +57,7 @@ function addGameListsArtwork() { #local tempGrid=$(generateGameLists_extraArtwork $file $platform) #local grid=$(echo "$tempGrid" | jq -r '.grid') - local vertical="$accountfolder/config/grid/emudeck/$platform/$file.jpg" + local vertical="$accountfolder/config/grid/retrolibrary/artwork/$platform/$file.jpg" local grid=$vertical local destination_vertical="$accountfolder/config/grid/${appID}p.png" #vertical local destination_hero="$accountfolder/config/grid/${appID}_hero.png" #BG @@ -70,7 +75,7 @@ function addGameListsArtwork() { generateGameLists_getPercentage() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/emudeck/" + local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" @@ -99,7 +104,7 @@ generateGameLists_extraArtwork() { local platform=$2 local hash=$3 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/emudeck" + local dest_folder="$accountfolder/config/grid/retrolibrary/artwork" wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game&hash=$hash" diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index dcceeca38..3a0e5736e 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -80,7 +80,7 @@ def collect_game_data(system_dir, extensions): rom_hash = calculate_hash(file_path) clean_name = name_cleaned - game_img = f"/customimages/emudeck/{platform}/{clean_name}.jpg" + game_img = f"//retrolibrary/artwork/{platform}/{clean_name}.jpg" game_info = { "name": clean_name, "filename": file_path, diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 0996c2328..1cfa49747 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -67,7 +67,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - game_img = f"/customimages/emudeck/{platform}/{name_cleaned}.jpg" + game_img = f"//retrolibrary/artwork/{platform}/{name_cleaned}.jpg" rom_hash = calculate_hash(file_path) # Verificar si la imagen existe en el images_path From 311acef8e48c6716540d23d9245b2944a9e384b5 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 01:08:23 +0100 Subject: [PATCH 084/230] proper path --- tools/retro-library/generate_game_lists.py | 2 +- tools/retro-library/missing_artwork.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 3a0e5736e..31dd96720 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -80,7 +80,7 @@ def collect_game_data(system_dir, extensions): rom_hash = calculate_hash(file_path) clean_name = name_cleaned - game_img = f"//retrolibrary/artwork/{platform}/{clean_name}.jpg" + game_img = f"/customimages/retrolibrary/artwork/{platform}/{clean_name}.jpg" game_info = { "name": clean_name, "filename": file_path, diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 1cfa49747..cff896634 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -67,7 +67,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - game_img = f"//retrolibrary/artwork/{platform}/{name_cleaned}.jpg" + game_img = f"/customimages/retrolibrary/artwork/{platform}/{name_cleaned}.jpg" rom_hash = calculate_hash(file_path) # Verificar si la imagen existe en el images_path From 1bf720973aadd4128e702b2bfa6cd80968cfc96c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 01:45:40 +0100 Subject: [PATCH 085/230] new paths --- tools/retro-library/generate_game_lists.py | 6 +++++- tools/retro-library/missing_artwork.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 31dd96720..9ba034083 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -80,12 +80,16 @@ def collect_game_data(system_dir, extensions): rom_hash = calculate_hash(file_path) clean_name = name_cleaned - game_img = f"/customimages/retrolibrary/artwork/{platform}/{clean_name}.jpg" + game_img = f"/customimages/retrolibrary/artwork/{platform}/box2dfront/{clean_name}.jpg" + game_ss = f"/customimages/retrolibrary/artwork/{platform}/screenshot/{clean_name}.jpg" + game_wheel = f"/customimages/retrolibrary/artwork/{platform}/wheel/{clean_name}.png" game_info = { "name": clean_name, "filename": file_path, "file": clean_name, "img": game_img, + "ss": game_ss, + "wheel": game_wheel, "platform": platform, "hash": rom_hash # Agregar el hash } diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index cff896634..938f5a3b8 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -67,7 +67,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - game_img = f"/customimages/retrolibrary/artwork/{platform}/{name_cleaned}.jpg" + game_img = f"/customimages/retrolibrary/artwork/{platform}/box2dfront/{name_cleaned}.jpg" rom_hash = calculate_hash(file_path) # Verificar si la imagen existe en el images_path From c590e15b414329ad2eb564f6ff187a516b011bb5 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 11:50:26 +0100 Subject: [PATCH 086/230] missing_artwork_platforms --- functions/generateGameLists.sh | 1 + .../missing_artwork_platforms.py | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tools/retro-library/missing_artwork_platforms.py diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 72ea034c3..7c5846465 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -33,6 +33,7 @@ generateGameLists_artwork() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" } diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py new file mode 100644 index 000000000..7ecbdc721 --- /dev/null +++ b/tools/retro-library/missing_artwork_platforms.py @@ -0,0 +1,68 @@ +import os +import json +import sys +import re + +def generate_systems_with_missing_images(roms_path, images_path): + def has_missing_images(system_dir, extensions): + for root, _, files in os.walk(system_dir): + for file in files: + file_path = os.path.join(root, file) + if os.path.islink(file_path): + continue + + filename = os.path.basename(file) + extension = filename.split('.')[-1] + name = '.'.join(filename.split('.')[:-1]) + if extension in extensions: + platform = os.path.basename(system_dir) + + name_cleaned = re.sub(r'\(.*?\)', '', name) + name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) + name_cleaned = name_cleaned.strip() + name_cleaned = name_cleaned.replace(' ', '_').replace('-', '_') + name_cleaned = re.sub(r'_+', '_', name_cleaned) + name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') + + img_path = os.path.join(images_path, f"{platform}/{name_cleaned}.jpg") + if not os.path.exists(img_path): + return True + return False + + roms_dir = roms_path + valid_system_dirs = [] + + for system_dir in os.listdir(roms_dir): + if system_dir == "xbox360": + system_dir = "xbox360/roms" + full_path = os.path.join(roms_dir, system_dir) + if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): + file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) + if file_count > 2: + valid_system_dirs.append(full_path) + + systems_with_missing_images = set() + + for system_dir in valid_system_dirs: + if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): + continue + + with open(os.path.join(system_dir, 'metadata.txt')) as f: + metadata = f.read() + extensions = next((line.split(':')[1].strip().replace(',', ' ') for line in metadata.splitlines() if line.startswith('extensions:')), '').split() + + if has_missing_images(system_dir, extensions): + systems_with_missing_images.add(os.path.basename(system_dir)) + + json_output = json.dumps(list(systems_with_missing_images), indent=4) + home_directory = os.path.expanduser("~") + output_file = os.path.join(home_directory, 'emudeck', 'cache', 'missing_systems.json') + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'w') as f: + f.write(json_output) + +# Pasar la ruta de las ROMs y de las imágenes desde los argumentos de línea de comandos +roms_path = sys.argv[1] +images_path = sys.argv[2] + +generate_systems_with_missing_images(roms_path, images_path) From 687861d4e8f9b046dbfd9271280ec505d56eff77 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 12:09:47 +0100 Subject: [PATCH 087/230] new media downloader --- functions/generateGameLists.sh | 6 +- tools/retro-library/download_art_platforms.py | 68 +++++++++++++++++++ tools/retro-library/generate_game_lists.py | 6 +- tools/retro-library/missing_artwork.py | 2 +- 4 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 tools/retro-library/download_art_platforms.py diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 7c5846465..ad2b24e18 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -34,7 +34,7 @@ generateGameLists_artwork() { local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" - python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" } @@ -43,7 +43,7 @@ saveImage(){ local name=$2 local system=$3 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/${system}" + local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/${system}/media/box2dfront/" local dest_path="$dest_folder/$name.jpg" wget -q -O "$dest_path" "$url" } @@ -58,7 +58,7 @@ function addGameListsArtwork() { #local tempGrid=$(generateGameLists_extraArtwork $file $platform) #local grid=$(echo "$tempGrid" | jq -r '.grid') - local vertical="$accountfolder/config/grid/retrolibrary/artwork/$platform/$file.jpg" + local vertical="$accountfolder/config/grid/retrolibrary/artwork/$platform/media/box2dfront/$file.jpg" local grid=$vertical local destination_vertical="$accountfolder/config/grid/${appID}p.png" #vertical local destination_hero="$accountfolder/config/grid/${appID}_hero.png" #BG diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py new file mode 100644 index 000000000..4496db5f6 --- /dev/null +++ b/tools/retro-library/download_art_platforms.py @@ -0,0 +1,68 @@ +import os +import json +import requests +import zipfile +from io import BytesIO +import sys + +def download_and_extract(output_dir): + # Ruta fija del archivo JSON + json_file_path = os.path.expanduser("~/emudeck/cache/missing_artwork.json") + + # Verificar que el archivo JSON exista + if not os.path.exists(json_file_path): + print(f"Archivo JSON no encontrado: {json_file_path}") + return + + # Leer el JSON + with open(json_file_path, 'r') as f: + try: + data = json.load(f) + except json.JSONDecodeError as e: + print(f"Error al leer el archivo JSON: {e}") + return + + # Extraer las plataformas incompletas + incomplete_platforms = data.get("incomplete_platforms", []) + if not isinstance(incomplete_platforms, list): + print("El archivo JSON no contiene una lista válida en 'incomplete_platforms'.") + return + + if not incomplete_platforms: + print("No hay plataformas incompletas en el archivo JSON.") + return + + # Crear el directorio de salida si no existe + os.makedirs(output_dir, exist_ok=True) + + # Procesar cada plataforma + for platform in incomplete_platforms: + url = f"https://bot.emudeck.com/artwork_deck/{platform}.zip" + print(f"Descargando: {url}") + + try: + # Descargar el archivo ZIP + response = requests.get(url, stream=True) + response.raise_for_status() # Lanza un error si la descarga falla + + # Leer el contenido del ZIP en memoria + with zipfile.ZipFile(BytesIO(response.content)) as zip_file: + print(f"Extrayendo contenido de {platform}.zip a {output_dir}") + zip_file.extractall(output_dir) # Sobrescribe por defecto + + except requests.exceptions.RequestException as e: + print(f"Error al descargar {url}: {e}") + except zipfile.BadZipFile as e: + print(f"Error al procesar el ZIP para {platform}: {e}") + + print("Proceso completado.") + +# Verificar argumentos de línea de comandos +if len(sys.argv) != 2: + print("Uso: python3 download_and_extract.py ") + sys.exit(1) + +# Directorio de salida pasado como argumento +output_dir = sys.argv[1] + +download_and_extract(output_dir) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 9ba034083..f7d693ddd 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -80,9 +80,9 @@ def collect_game_data(system_dir, extensions): rom_hash = calculate_hash(file_path) clean_name = name_cleaned - game_img = f"/customimages/retrolibrary/artwork/{platform}/box2dfront/{clean_name}.jpg" - game_ss = f"/customimages/retrolibrary/artwork/{platform}/screenshot/{clean_name}.jpg" - game_wheel = f"/customimages/retrolibrary/artwork/{platform}/wheel/{clean_name}.png" + game_img = f"/customimages/retrolibrary/artwork/{platform}/media/box2dfront/{clean_name}.jpg" + game_ss = f"/customimages/retrolibrary/artwork/{platform}/media/screenshot/{clean_name}.jpg" + game_wheel = f"/customimages/retrolibrary/artwork/{platform}/media/wheel/{clean_name}.png" game_info = { "name": clean_name, "filename": file_path, diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 938f5a3b8..c81ec56f9 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -67,7 +67,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - game_img = f"/customimages/retrolibrary/artwork/{platform}/box2dfront/{name_cleaned}.jpg" + game_img = f"/customimages/retrolibrary/artwork/{platform}/media/box2dfront/{name_cleaned}.jpg" rom_hash = calculate_hash(file_path) # Verificar si la imagen existe en el images_path From 7ea590e163c0db8bbae6f3e8d248b0b874943f2b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 19:29:00 +0100 Subject: [PATCH 088/230] symlinks --- functions/generateGameLists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index ad2b24e18..2bdb6688f 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -7,8 +7,8 @@ generateGameLists() { mkdir -p "$storagePath/retrolibrary/data" mkdir -p "$accountfolder/config/grid/retrolibrary" - ln -s "$storagePath/retrolibrary/artwork" "$accountfolder/config/grid/retrolibrary/artwork" - ln -s "$storagePath/retrolibrary/data" "$accountfolder/config/grid/retrolibrary/data" + ln -s "$storagePath/retrolibrary/artwork/" "$accountfolder/config/grid/retrolibrary/artwork/" + ln -s "$storagePath/retrolibrary/data/" "$accountfolder/config/grid/retrolibrary/data/" pegasus_setPaths rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$dest_folder" --keep-dirlinks From 5d2eb8ed48a2b538cec7fe5f664e384c333047e8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 19:37:51 +0100 Subject: [PATCH 089/230] download data + achievements --- functions/generateGameLists.sh | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 2bdb6688f..19de747e5 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -4,11 +4,11 @@ generateGameLists() { local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" mkdir -p "$storagePath/retrolibrary/artwork" - mkdir -p "$storagePath/retrolibrary/data" mkdir -p "$accountfolder/config/grid/retrolibrary" - ln -s "$storagePath/retrolibrary/artwork/" "$accountfolder/config/grid/retrolibrary/artwork/" - ln -s "$storagePath/retrolibrary/data/" "$accountfolder/config/grid/retrolibrary/data/" + + generateGameLists_downloadAchievements + generateGameLists_downloadData pegasus_setPaths rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$dest_folder" --keep-dirlinks @@ -125,4 +125,23 @@ generateGameLists_retroAchievements(){ local hash=$1 local systemID=$2 python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" "$systemID" -} \ No newline at end of file +} + +generateGameLists_downloadAchievements(){ + local folder="$storagePath/retrolibrary/achievements" + if [ ! -d $folder ]; then + mkdip -p $folder + wget -q -O "$folder" "https://bot.emudeck.com/achievements/achievements.zip" + cd $folder && unzip -o achievements.zip && rm achievements.zip + fi +} + +generateGameLists_downloadData(){ + local folder="$storagePath/retrolibrary/data" + if [ ! -d $folder ]; then + mkdip -p $folder + wget -q -O "$folder" "https://bot.emudeck.com/data/data.zip" + cd $folder && unzip -o data.zip && rm data.zip + fi +} + From 446db1ca59ff662cfd60de2721942a16acf604e8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 19:46:34 +0100 Subject: [PATCH 090/230] local RA --- functions/EmuScripts/emuDeckRetroArch.sh | 40 +++++++++++------------ functions/generateGameLists.sh | 5 +-- tools/retro-library/retro_achievements.py | 4 ++- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/functions/EmuScripts/emuDeckRetroArch.sh b/functions/EmuScripts/emuDeckRetroArch.sh index 73563c356..c52bebe1d 100644 --- a/functions/EmuScripts/emuDeckRetroArch.sh +++ b/functions/EmuScripts/emuDeckRetroArch.sh @@ -39,7 +39,7 @@ RetroArch_backupConfigs(){ #Install RetroArch_install(){ - setMSG "Installing $RetroArch_emuName" + setMSG "Installing $RetroArch_emuName" installEmuFP "${RetroArch_emuName}" "${RetroArch_emuPath}" "emulator" "" RetroArch_installCores } @@ -1798,15 +1798,15 @@ RetroArch_Mupen64Plus_Next_setUpCoreOpt(){ RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-virefresh' '"Auto"' # hd pack settings - # Commenting these out. These seem to be causing a lot of graphical issues. + # Commenting these out. These seem to be causing a lot of graphical issues. #RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-txHiresEnable' '"True"' #RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-txHiresFullAlphaChannel' '"True"' #RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-txCacheCompression' '"True"' #RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-EnableEnhancedHighResStorage' '"True"' #RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-EnableEnhancedTextureStorage' '"False"' # lazy loading - + # revert hd pack settings - # These seem to be causing a lot of graphical issues. + # These seem to be causing a lot of graphical issues. RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-txHiresEnable' '"False"' RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-txHiresFullAlphaChannel' '"False"' RetroArch_setOverride 'Mupen64Plus-Next.opt' 'Mupen64Plus-Next' 'mupen64plus-txCacheCompression' '"False"' @@ -2476,7 +2476,7 @@ RetroArch_retroAchievementsSetLogin(){ iniFieldUpdate "$RetroArch_configFile" "" "cheevos_username" "$rau" iniFieldUpdate "$RetroArch_configFile" "" "cheevos_token" "$rat" - + setSetting cheevos_username $rau fi } RetroArch_setBezels(){ @@ -2547,18 +2547,18 @@ RetroArch_Mupen64Plus_Next_setUpHdPacks(){ local texturePackPath="$HOME/.var/app/org.libretro.RetroArch/config/retroarch/system/Mupen64plus/hires_texture" local textureCachePath="$HOME/.var/app/org.libretro.RetroArch/config/retroarch/system/Mupen64plus/cache" - # Something in the install is causng infinite symlinks, commenting these lines out for now and deleting folders. Needs more thorough testing. + # Something in the install is causng infinite symlinks, commenting these lines out for now and deleting folders. Needs more thorough testing. if [[ -L "$biosPath/Mupen64plus/cache/cache" || -d "$biosPath/Mupen64plus/cache/cache" ]]; then - rm -rf "$biosPath/Mupen64plus/cache/" + rm -rf "$biosPath/Mupen64plus/cache/" fi - if [[ -L "$emulationPath/hdpacks/n64" || -d "$emulationPath/hdpacks/n64" ]]; then + if [[ -L "$emulationPath/hdpacks/n64" || -d "$emulationPath/hdpacks/n64" ]]; then rm -rf "$emulationPath/hdpacks/n64" - fi + fi - if [[ -d "$emulationPath/hdpacks/retroarch" ]]; then - rm -rf "$emulationPath/hdpacks/retroarch" - fi + if [[ -d "$emulationPath/hdpacks/retroarch" ]]; then + rm -rf "$emulationPath/hdpacks/retroarch" + fi #mkdir -p "$texturePackPath" #mkdir -p "$textureCachePath" @@ -2576,22 +2576,22 @@ RetroArch_Mupen64Plus_Next_setUpHdPacks(){ # setupHdPacks() RetroArch_MesenNES_setUpHdPacks(){ - # Something in the install is causng infinite symlinks, commenting these lines out for now and deleting folders. Needs more thorough testing. + # Something in the install is causng infinite symlinks, commenting these lines out for now and deleting folders. Needs more thorough testing. if [[ -L "$biosPath/HdPacks/HdPacks" || -d "$biosPath/HdPacks/HdPacks" ]]; then - rm -rf "$biosPath/HdPacks" + rm -rf "$biosPath/HdPacks" fi if [[ -L "$biosPath/hdpacks/nes/HdPacks/HdPacks" || -d "$biosPath/hdpacks/nes/HdPacks/HdPacks" ]]; then - rm -rf "$biosPath/hdpacks/nes" + rm -rf "$biosPath/hdpacks/nes" fi - if [ -L "$emulationPath/hdpacks/nes" ]; then + if [ -L "$emulationPath/hdpacks/nes" ]; then rm -rf "$emulationPath/hdpacks/nes" - fi + fi - if [[ -d "$emulationPath/hdpacks/retroarch" ]]; then - rm -rf "$emulationPath/hdpacks/retroarch" - fi + if [[ -d "$emulationPath/hdpacks/retroarch" ]]; then + rm -rf "$emulationPath/hdpacks/retroarch" + fi #NES #unlink "$emulationPath"/hdpacks/Mesen 2>/dev/null #refresh link if moved diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 19de747e5..53ebe68e1 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -123,8 +123,9 @@ generateGameLists_extraArtwork() { generateGameLists_retroAchievements(){ local hash=$1 - local systemID=$2 - python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" "$systemID" + local system=$2 + local localDataPath="$storagePath/retrolibrary/achievements/$system.json" + python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" "$system" "$localDataPath" } generateGameLists_downloadAchievements(){ diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py index 700852dff..51f5116c9 100644 --- a/tools/retro-library/retro_achievements.py +++ b/tools/retro-library/retro_achievements.py @@ -12,9 +12,11 @@ USER = sys.argv[1] md5_to_find = sys.argv[2] SYSTEM_ID = sys.argv[3] +LOCAL_PATH = sys.argv[4] + # Endpoints BASE_URL = "https://retroachievements.org/API/" -GAMES_LIST_ENDPOINT = f"{BASE_URL}API_GetGameList.php" +GAMES_LIST_ENDPOINT = f"{LOCAL_PATH}" GAME_INFO_ENDPOINT = f"{BASE_URL}API_GetGameInfoAndUserProgress.php" # Función para obtener la lista de juegos del sistema From 7c373498edcedd186a2a8ee7beeb78e7ccbf807c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 19:54:24 +0100 Subject: [PATCH 091/230] fix --- tools/retro-library/retro_achievements.py | 27 ++++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py index 51f5116c9..166dccedd 100644 --- a/tools/retro-library/retro_achievements.py +++ b/tools/retro-library/retro_achievements.py @@ -21,18 +21,23 @@ # Función para obtener la lista de juegos del sistema def get_games(system_id): - response = requests.get(GAMES_LIST_ENDPOINT, params={ - "i": system_id, - "h": 1, - "f": 1, - "z": API_USERNAME, - "y": API_KEY - }) - if response.status_code == 200: - return response.json() - else: - print(f"Error al obtener los juegos: {response.status_code}") + try: + # Abrir el archivo local + with open(GAMES_LIST_ENDPOINT, 'r') as file: + # Leer y cargar el contenido JSON + data = json.load(file) + + # Filtrar los juegos por el `system_id` si es necesario + filtered_games = [game for game in data if game.get("system_id") == system_id] + + return filtered_games + except FileNotFoundError: + print(f"Error: El archivo {GAMES_LIST_ENDPOINT} no se encuentra.") return None + except json.JSONDecodeError: + print(f"Error: El archivo {GAMES_LIST_ENDPOINT} no contiene un JSON válido.") + return None + # Función para obtener información del juego y progreso del usuario def get_game_info_and_progress(game_id): From fd17b85a5eb96aeb329f6de65099d5a36d0a0284 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 20:00:06 +0100 Subject: [PATCH 092/230] fix --- functions/generateGameLists.sh | 2 +- tools/retro-library/retro_achievements.py | 20 +++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 53ebe68e1..062999e5b 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -125,7 +125,7 @@ generateGameLists_retroAchievements(){ local hash=$1 local system=$2 local localDataPath="$storagePath/retrolibrary/achievements/$system.json" - python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" "$system" "$localDataPath" + python $HOME/.config/EmuDeck/backend/tools/retro-library/retro_achievements.py "$cheevos_username" "$hash" "$localDataPath" } generateGameLists_downloadAchievements(){ diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py index 166dccedd..9d3a7e0c7 100644 --- a/tools/retro-library/retro_achievements.py +++ b/tools/retro-library/retro_achievements.py @@ -11,26 +11,21 @@ API_KEY = "mvLqoKB3JmbXrezCd7LIXzMnV42ApWzj" USER = sys.argv[1] md5_to_find = sys.argv[2] -SYSTEM_ID = sys.argv[3] -LOCAL_PATH = sys.argv[4] +LOCAL_PATH = sys.argv[3] # Endpoints BASE_URL = "https://retroachievements.org/API/" GAMES_LIST_ENDPOINT = f"{LOCAL_PATH}" GAME_INFO_ENDPOINT = f"{BASE_URL}API_GetGameInfoAndUserProgress.php" -# Función para obtener la lista de juegos del sistema -def get_games(system_id): +# Función para cargar la lista de juegos +def get_games(): try: - # Abrir el archivo local with open(GAMES_LIST_ENDPOINT, 'r') as file: # Leer y cargar el contenido JSON data = json.load(file) - - # Filtrar los juegos por el `system_id` si es necesario - filtered_games = [game for game in data if game.get("system_id") == system_id] - - return filtered_games + print(f"Loaded {len(data)} games.") # Mensaje de depuración + return data except FileNotFoundError: print(f"Error: El archivo {GAMES_LIST_ENDPOINT} no se encuentra.") return None @@ -38,7 +33,6 @@ def get_games(system_id): print(f"Error: El archivo {GAMES_LIST_ENDPOINT} no contiene un JSON válido.") return None - # Función para obtener información del juego y progreso del usuario def get_game_info_and_progress(game_id): response = requests.get(GAME_INFO_ENDPOINT, params={ @@ -56,7 +50,7 @@ def get_game_info_and_progress(game_id): # Función principal def main(): # Obtener lista de juegos - games = get_games(SYSTEM_ID) + games = get_games() if not games: return @@ -64,7 +58,7 @@ def main(): game_id = None for game in games: hashes = game.get("Hashes", []) - if md5_to_find in hashes: + if isinstance(hashes, list) and md5_to_find in hashes: game_id = game.get("ID") break From e4ac9c9e4d241c95611e7154788a4778ff3e64d3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 21 Nov 2024 20:03:27 +0100 Subject: [PATCH 093/230] no print --- tools/retro-library/retro_achievements.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/retro-library/retro_achievements.py b/tools/retro-library/retro_achievements.py index 9d3a7e0c7..f819b1edf 100644 --- a/tools/retro-library/retro_achievements.py +++ b/tools/retro-library/retro_achievements.py @@ -24,7 +24,6 @@ def get_games(): with open(GAMES_LIST_ENDPOINT, 'r') as file: # Leer y cargar el contenido JSON data = json.load(file) - print(f"Loaded {len(data)} games.") # Mensaje de depuración return data except FileNotFoundError: print(f"Error: El archivo {GAMES_LIST_ENDPOINT} no se encuentra.") From 3c79247810ac31c3bd3b859ed828e8f430b21e28 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 11:41:52 +0100 Subject: [PATCH 094/230] missing artwork more types --- tools/retro-library/missing_artwork.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index c81ec56f9..056e5e217 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -15,6 +15,16 @@ def calculate_hash(file_path): except Exception as e: print(f"Error al calcular el hash para {file_path}: {e}") return None + + def missing_images(images_path, platform, name_cleaned, folders): + """Devuelve una lista de carpetas donde falta la imagen.""" + missing = [] + for folder in folders: + img_path = os.path.join(images_path, f"{platform}/{folder}/{name_cleaned}.jpg") + if not os.path.exists(img_path): + missing.append(folder) + return missing + def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -67,17 +77,15 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - game_img = f"/customimages/retrolibrary/artwork/{platform}/media/box2dfront/{name_cleaned}.jpg" - rom_hash = calculate_hash(file_path) - # Verificar si la imagen existe en el images_path - - img_path = os.path.join(images_path, f"{platform}/{name_cleaned}.jpg") - #print(img_path) - if not os.path.exists(img_path): + # Verificar imágenes faltantes + missing = missing_images(images_path, platform, name_cleaned, ["box2dfront", "wheel", "screenshot"]) + if missing: + rom_hash = calculate_hash(file_path) game_info = { "name": name_cleaned, "platform": platform, - "hash": rom_hash + "hash": rom_hash, + "missing_images": missing } game_data.append(game_info) @@ -121,4 +129,3 @@ def collect_game_data(system_dir, extensions): roms_path = sys.argv[1] images_path = sys.argv[2] -generate_game_lists(roms_path, images_path) From c1087417c9145e5fa11bd0487467671464312a44 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 11:45:05 +0100 Subject: [PATCH 095/230] media --- functions/generateGameLists.sh | 3 ++- tools/retro-library/missing_artwork.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 062999e5b..edcbed58b 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -32,10 +32,11 @@ generateGameLists_artwork() { mkdir -p "$HOME/emudeck/cache/" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" } saveImage(){ diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 056e5e217..7c35435c6 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -20,7 +20,7 @@ def missing_images(images_path, platform, name_cleaned, folders): """Devuelve una lista de carpetas donde falta la imagen.""" missing = [] for folder in folders: - img_path = os.path.join(images_path, f"{platform}/{folder}/{name_cleaned}.jpg") + img_path = os.path.join(images_path, f"{platform}/media/{folder}/{name_cleaned}.jpg") if not os.path.exists(img_path): missing.append(folder) return missing From 31a2813471a310a020ec919902d8bed77aedada3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 12:04:06 +0100 Subject: [PATCH 096/230] media download 1 by 1 --- tools/retro-library/download_art.py | 18 ++++++---- tools/retro-library/missing_artwork.py | 46 +++++++++++++++++--------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index b7ba14e7f..5b2a47854 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -19,11 +19,14 @@ def create_empty_image(name, platform, save_folder): pass # No escribimos nada para que quede vacío print(f"Archivo vacío creado para {name}") -def download_image(name, platform, img_url, save_folder): +def download_image(name, platform, img_url, save_folder, type): # Crear la carpeta si no existe os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado - img_path = os.path.join(save_folder, f"{platform}/{name}.jpg") + extension = "jpg" + if type === "wheel" + extension = "png" + img_path = os.path.join(save_folder, f"{platform}/media/{type}/{name}.{extension}") print(img_path) # Descargar y guardar la imagen response = requests.get(img_url) @@ -38,6 +41,7 @@ def fetch_image_data(game): name = game['name'] platform = game['platform'] hash = game['hash'] + type = game['type'] url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}&hash={hash}" try: @@ -46,17 +50,17 @@ def fetch_image_data(game): data = response.json() img_url = data.get('img') # Usar get para evitar errores si 'img' no existe o es None if img_url: - create_empty_image(name, platform, save_folder) - download_image(name, platform, img_url, save_folder) + create_empty_image(name, platform, save_folder, type) + download_image(name, platform, img_url, save_folder, type) else: print(f"No se encontró una URL de imagen válida para {platform}/{name}. Creando archivo vacío.") - create_empty_image(name, platform, save_folder) + create_empty_image(name, platform, save_folder, type) else: print(f"Error al llamar al servicio para {platform}/{name}") - create_empty_image(name, platform, save_folder) + create_empty_image(name, platform, save_folder, type) except requests.RequestException as e: print(f"Excepción al procesar {platform}/{name}: {e}") - create_empty_image(name, platform, save_folder) + create_empty_image(name, platform, save_folder, type) def process_json(save_folder): # Leer el JSON original diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 7c35435c6..a5c3b3988 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -15,16 +15,6 @@ def calculate_hash(file_path): except Exception as e: print(f"Error al calcular el hash para {file_path}: {e}") return None - - def missing_images(images_path, platform, name_cleaned, folders): - """Devuelve una lista de carpetas donde falta la imagen.""" - missing = [] - for folder in folders: - img_path = os.path.join(images_path, f"{platform}/media/{folder}/{name_cleaned}.jpg") - if not os.path.exists(img_path): - missing.append(folder) - return missing - def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -77,17 +67,42 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - # Verificar imágenes faltantes - missing = missing_images(images_path, platform, name_cleaned, ["box2dfront", "wheel", "screenshot"]) - if missing: - rom_hash = calculate_hash(file_path) + rom_hash = calculate_hash(file_path) + # Verificar si la imagen existe en el images_path + + img_path = os.path.join(images_path, f"{platform}/media/box2dfront/{name_cleaned}.jpg") + + #print(img_path) + if not os.path.exists(img_path): + game_info = { + "name": name_cleaned, + "platform": platform, + "hash": rom_hash, + "type": "box2dfront" + } + game_data.append(game_info) + + img_path = os.path.join(images_path, f"{platform}/media/wheel/{name_cleaned}.png") + print(img_path) + if not os.path.exists(img_path): game_info = { "name": name_cleaned, "platform": platform, "hash": rom_hash, - "missing_images": missing + "type": "wheel" } game_data.append(game_info) + img_path = os.path.join(images_path, f"{platform}/media/screenshot/{name_cleaned}.jpg") + #print(img_path) + if not os.path.exists(img_path): + game_info = { + "name": name_cleaned, + "platform": platform, + "hash": rom_hash, + "type": "screenshot" + } + game_data.append(game_info) + game_data_sorted = sorted(game_data, key=lambda x: x['name']) return game_data_sorted @@ -129,3 +144,4 @@ def collect_game_data(system_dir, extensions): roms_path = sys.argv[1] images_path = sys.argv[2] +generate_game_lists(roms_path, images_path) From 68261e1e9afdbb56c1258b5c6cd28a30ec11a5f9 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 12:06:21 +0100 Subject: [PATCH 097/230] no print --- tools/retro-library/download_art_platforms.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 4496db5f6..c658d4973 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -11,7 +11,7 @@ def download_and_extract(output_dir): # Verificar que el archivo JSON exista if not os.path.exists(json_file_path): - print(f"Archivo JSON no encontrado: {json_file_path}") + #print(f"Archivo JSON no encontrado: {json_file_path}") return # Leer el JSON @@ -19,17 +19,17 @@ def download_and_extract(output_dir): try: data = json.load(f) except json.JSONDecodeError as e: - print(f"Error al leer el archivo JSON: {e}") + #print(f"Error al leer el archivo JSON: {e}") return # Extraer las plataformas incompletas incomplete_platforms = data.get("incomplete_platforms", []) if not isinstance(incomplete_platforms, list): - print("El archivo JSON no contiene una lista válida en 'incomplete_platforms'.") + #print("El archivo JSON no contiene una lista válida en 'incomplete_platforms'.") return if not incomplete_platforms: - print("No hay plataformas incompletas en el archivo JSON.") + #print("No hay plataformas incompletas en el archivo JSON.") return # Crear el directorio de salida si no existe @@ -38,7 +38,7 @@ def download_and_extract(output_dir): # Procesar cada plataforma for platform in incomplete_platforms: url = f"https://bot.emudeck.com/artwork_deck/{platform}.zip" - print(f"Descargando: {url}") + #print(f"Descargando: {url}") try: # Descargar el archivo ZIP @@ -47,19 +47,19 @@ def download_and_extract(output_dir): # Leer el contenido del ZIP en memoria with zipfile.ZipFile(BytesIO(response.content)) as zip_file: - print(f"Extrayendo contenido de {platform}.zip a {output_dir}") + #print(f"Extrayendo contenido de {platform}.zip a {output_dir}") zip_file.extractall(output_dir) # Sobrescribe por defecto except requests.exceptions.RequestException as e: - print(f"Error al descargar {url}: {e}") + #print(f"Error al descargar {url}: {e}") except zipfile.BadZipFile as e: - print(f"Error al procesar el ZIP para {platform}: {e}") + #print(f"Error al procesar el ZIP para {platform}: {e}") - print("Proceso completado.") + #print("Proceso completado.") # Verificar argumentos de línea de comandos if len(sys.argv) != 2: - print("Uso: python3 download_and_extract.py ") + #print("Uso: python3 download_and_extract.py ") sys.exit(1) # Directorio de salida pasado como argumento From 738d67f485be69058f022ed74b7f63df8371f488 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 12:10:32 +0100 Subject: [PATCH 098/230] fix if --- tools/retro-library/download_art.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 5b2a47854..78b02e2a2 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -8,11 +8,14 @@ save_folder = sys.argv[1] json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork.json') -def create_empty_image(name, platform, save_folder): +def create_empty_image(name, platform, save_folder, type): + extension = "jpg" + if type == "wheel": + extension = "png" # Crear la carpeta si no existe os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado para el archivo vacío - img_path = os.path.join(save_folder, f"{platform}/{name}.jpg") + img_path = os.path.join(save_folder, f"{platform}/media/{type}/{name}.{extension}") print(img_path) # Crear un archivo vacío with open(img_path, 'wb') as file: @@ -24,7 +27,7 @@ def download_image(name, platform, img_url, save_folder, type): os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado extension = "jpg" - if type === "wheel" + if type == "wheel": extension = "png" img_path = os.path.join(save_folder, f"{platform}/media/{type}/{name}.{extension}") print(img_path) From 11a80d8f3aaccdc2fb8116bfcd43efafff42e1d7 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 13:01:36 +0100 Subject: [PATCH 099/230] some fixes --- functions/generateGameLists.sh | 14 ++++----- tools/retro-library/download_art.py | 13 +++----- tools/retro-library/download_art_platforms.py | 31 +++++++++---------- tools/retro-library/missing_artwork.py | 8 ++--- 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index edcbed58b..ce8d08893 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -4,8 +4,8 @@ generateGameLists() { local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" mkdir -p "$storagePath/retrolibrary/artwork" - mkdir -p "$accountfolder/config/grid/retrolibrary" - ln -s "$storagePath/retrolibrary/artwork/" "$accountfolder/config/grid/retrolibrary/artwork/" + mkdir -p "$accountfolder/config/grid/retrolibrary/" + ln -s "$storagePath/retrolibrary/artwork/" "$accountfolder/config/grid/retrolibrary/artwork" generateGameLists_downloadAchievements generateGameLists_downloadData @@ -132,17 +132,17 @@ generateGameLists_retroAchievements(){ generateGameLists_downloadAchievements(){ local folder="$storagePath/retrolibrary/achievements" if [ ! -d $folder ]; then - mkdip -p $folder - wget -q -O "$folder" "https://bot.emudeck.com/achievements/achievements.zip" - cd $folder && unzip -o achievements.zip && rm achievements.zip + mkdir -p $folder + wget -q -O "$folder/achievements.zip" "https://bot.emudeck.com/achievements/achievements.zip" + cd "$folder" && unzip -o achievements.zip && rm achievements.zip fi } generateGameLists_downloadData(){ local folder="$storagePath/retrolibrary/data" if [ ! -d $folder ]; then - mkdip -p $folder - wget -q -O "$folder" "https://bot.emudeck.com/data/data.zip" + mkdir -p $folder + wget -q -O "$folder/data.zip" "https://bot.emudeck.com/data/data.zip" cd $folder && unzip -o data.zip && rm data.zip fi } diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 78b02e2a2..b21bfa0af 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -8,10 +8,7 @@ save_folder = sys.argv[1] json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork.json') -def create_empty_image(name, platform, save_folder, type): - extension = "jpg" - if type == "wheel": - extension = "png" +def create_empty_image(name, platform, save_folder): # Crear la carpeta si no existe os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado para el archivo vacío @@ -53,17 +50,17 @@ def fetch_image_data(game): data = response.json() img_url = data.get('img') # Usar get para evitar errores si 'img' no existe o es None if img_url: - create_empty_image(name, platform, save_folder, type) + #create_empty_image(name, platform, save_folder, type) download_image(name, platform, img_url, save_folder, type) else: print(f"No se encontró una URL de imagen válida para {platform}/{name}. Creando archivo vacío.") - create_empty_image(name, platform, save_folder, type) + #create_empty_image(name, platform, save_folder, type) else: print(f"Error al llamar al servicio para {platform}/{name}") - create_empty_image(name, platform, save_folder, type) + #create_empty_image(name, platform, save_folder, type) except requests.RequestException as e: print(f"Excepción al procesar {platform}/{name}: {e}") - create_empty_image(name, platform, save_folder, type) + #create_empty_image(name, platform, save_folder, type) def process_json(save_folder): # Leer el JSON original diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index c658d4973..1a09683c7 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -7,11 +7,11 @@ def download_and_extract(output_dir): # Ruta fija del archivo JSON - json_file_path = os.path.expanduser("~/emudeck/cache/missing_artwork.json") + json_file_path = os.path.expanduser("~/emudeck/cache/missing_systems.json") # Verificar que el archivo JSON exista if not os.path.exists(json_file_path): - #print(f"Archivo JSON no encontrado: {json_file_path}") + print(f"Archivo JSON no encontrado: {json_file_path}") return # Leer el JSON @@ -19,26 +19,25 @@ def download_and_extract(output_dir): try: data = json.load(f) except json.JSONDecodeError as e: - #print(f"Error al leer el archivo JSON: {e}") + print(f"Error al leer el archivo JSON: {e}") return - # Extraer las plataformas incompletas - incomplete_platforms = data.get("incomplete_platforms", []) - if not isinstance(incomplete_platforms, list): - #print("El archivo JSON no contiene una lista válida en 'incomplete_platforms'.") + # Verificar que el JSON sea una lista + if not isinstance(data, list): + print("El archivo JSON no contiene una lista válida.") return - if not incomplete_platforms: - #print("No hay plataformas incompletas en el archivo JSON.") + if not data: + print("No hay plataformas en el archivo JSON.") return # Crear el directorio de salida si no existe os.makedirs(output_dir, exist_ok=True) # Procesar cada plataforma - for platform in incomplete_platforms: + for platform in data: url = f"https://bot.emudeck.com/artwork_deck/{platform}.zip" - #print(f"Descargando: {url}") + print(f"Descargando: {url}") try: # Descargar el archivo ZIP @@ -47,19 +46,19 @@ def download_and_extract(output_dir): # Leer el contenido del ZIP en memoria with zipfile.ZipFile(BytesIO(response.content)) as zip_file: - #print(f"Extrayendo contenido de {platform}.zip a {output_dir}") + print(f"Extrayendo contenido de {platform}.zip a {output_dir}") zip_file.extractall(output_dir) # Sobrescribe por defecto except requests.exceptions.RequestException as e: - #print(f"Error al descargar {url}: {e}") + print(f"Error al descargar {url}: {e}") except zipfile.BadZipFile as e: - #print(f"Error al procesar el ZIP para {platform}: {e}") + print(f"Error al procesar el ZIP para {platform}: {e}") - #print("Proceso completado.") + print("Proceso completado.") # Verificar argumentos de línea de comandos if len(sys.argv) != 2: - #print("Uso: python3 download_and_extract.py ") + print("Uso: python3 download_and_extract.py ") sys.exit(1) # Directorio de salida pasado como argumento diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index a5c3b3988..ae6426772 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -13,7 +13,7 @@ def calculate_hash(file_path): hash_md5.update(chunk) return hash_md5.hexdigest() except Exception as e: - print(f"Error al calcular el hash para {file_path}: {e}") + #print(f"Error al calcular el hash para {file_path}: {e}") return None def collect_game_data(system_dir, extensions): game_data = [] @@ -72,7 +72,7 @@ def collect_game_data(system_dir, extensions): img_path = os.path.join(images_path, f"{platform}/media/box2dfront/{name_cleaned}.jpg") - #print(img_path) + ##print(img_path) if not os.path.exists(img_path): game_info = { "name": name_cleaned, @@ -83,7 +83,7 @@ def collect_game_data(system_dir, extensions): game_data.append(game_info) img_path = os.path.join(images_path, f"{platform}/media/wheel/{name_cleaned}.png") - print(img_path) + #print(img_path) if not os.path.exists(img_path): game_info = { "name": name_cleaned, @@ -93,7 +93,7 @@ def collect_game_data(system_dir, extensions): } game_data.append(game_info) img_path = os.path.join(images_path, f"{platform}/media/screenshot/{name_cleaned}.jpg") - #print(img_path) + ##print(img_path) if not os.path.exists(img_path): game_info = { "name": name_cleaned, From 09a6c0d283927c73bbced63900c9fa9d346b81f4 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 13:06:55 +0100 Subject: [PATCH 100/230] fixxxxxes --- tools/retro-library/download_art.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index b21bfa0af..9387c47b8 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -8,7 +8,10 @@ save_folder = sys.argv[1] json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork.json') -def create_empty_image(name, platform, save_folder): +def create_empty_image(name, platform, save_folder, type): + extension = "jpg" + if type == "wheel": + extension = "png" # Crear la carpeta si no existe os.makedirs(save_folder, exist_ok=True) # Definir la ruta de guardado para el archivo vacío @@ -54,13 +57,13 @@ def fetch_image_data(game): download_image(name, platform, img_url, save_folder, type) else: print(f"No se encontró una URL de imagen válida para {platform}/{name}. Creando archivo vacío.") - #create_empty_image(name, platform, save_folder, type) + create_empty_image(name, platform, save_folder, type) else: print(f"Error al llamar al servicio para {platform}/{name}") - #create_empty_image(name, platform, save_folder, type) + create_empty_image(name, platform, save_folder, type) except requests.RequestException as e: print(f"Excepción al procesar {platform}/{name}: {e}") - #create_empty_image(name, platform, save_folder, type) + create_empty_image(name, platform, save_folder, type) def process_json(save_folder): # Leer el JSON original From d16b8a080460bc23c70e18e4c8acc62721cab97a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 14:09:41 +0100 Subject: [PATCH 101/230] =?UTF-8?q?contando=20n=C3=BAmero=20de=20box2d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/generateGameLists.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index ce8d08893..85c1a595d 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -87,8 +87,7 @@ generateGameLists_getPercentage() { # Contar el número total de juegos en `roms_games.json` local games=$(jq '[.[].games[]] | length' "$json_file") - - local artwork_missing=$(jq '. | length' "$json_file_artwork") + local artwork_missing=$(jq '[.[] | select(.type == "box2dart")] | length' "$json_file_artwork") if [[ -z "$games" || "$games" -eq 0 ]]; then return From ccf570eb17537918531f9802e12479532573a26c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 14:14:35 +0100 Subject: [PATCH 102/230] create_empty_image --- tools/retro-library/download_art.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 9387c47b8..78b02e2a2 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -53,7 +53,7 @@ def fetch_image_data(game): data = response.json() img_url = data.get('img') # Usar get para evitar errores si 'img' no existe o es None if img_url: - #create_empty_image(name, platform, save_folder, type) + create_empty_image(name, platform, save_folder, type) download_image(name, platform, img_url, save_folder, type) else: print(f"No se encontró una URL de imagen válida para {platform}/{name}. Creando archivo vacío.") From 1b4017d2bbe70cfcb66cd2b05b3230fbda81b1eb Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 14:18:55 +0100 Subject: [PATCH 103/230] retry network failure --- tools/retro-library/download_art.py | 75 ++++++++++++++--------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 78b02e2a2..6593e935c 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -6,39 +6,34 @@ # Path del JSON y carpeta de destino desde los argumentos save_folder = sys.argv[1] -json_path = os.path.expanduser(f'~/emudeck/cache/missing_artwork.json') +json_path = os.path.expanduser('~/emudeck/cache/missing_artwork.json') def create_empty_image(name, platform, save_folder, type): - extension = "jpg" - if type == "wheel": - extension = "png" - # Crear la carpeta si no existe - os.makedirs(save_folder, exist_ok=True) - # Definir la ruta de guardado para el archivo vacío - img_path = os.path.join(save_folder, f"{platform}/media/{type}/{name}.{extension}") - print(img_path) - # Crear un archivo vacío + extension = "jpg" if type != "wheel" else "png" + # Crear la ruta completa para la carpeta + folder_path = os.path.join(save_folder, platform, "media", type) + os.makedirs(folder_path, exist_ok=True) + img_path = os.path.join(folder_path, f"{name}.{extension}") + # Crear archivo vacío with open(img_path, 'wb') as file: - pass # No escribimos nada para que quede vacío - print(f"Archivo vacío creado para {name}") + pass + print(f"Archivo vacío creado: {img_path}") def download_image(name, platform, img_url, save_folder, type): - # Crear la carpeta si no existe - os.makedirs(save_folder, exist_ok=True) - # Definir la ruta de guardado - extension = "jpg" - if type == "wheel": - extension = "png" - img_path = os.path.join(save_folder, f"{platform}/media/{type}/{name}.{extension}") - print(img_path) - # Descargar y guardar la imagen - response = requests.get(img_url) - if response.status_code == 200: + extension = "jpg" if type != "wheel" else "png" + # Crear la ruta completa para la carpeta + folder_path = os.path.join(save_folder, platform, "media", type) + os.makedirs(folder_path, exist_ok=True) + img_path = os.path.join(folder_path, f"{name}.{extension}") + try: + response = requests.get(img_url, timeout=10) + response.raise_for_status() # Lanza una excepción para códigos de error HTTP with open(img_path, 'wb') as file: file.write(response.content) - print(f"Imagen guardada como {img_path}") - else: - print(f"Error al descargar la imagen para {platform}/{name}") + print(f"Imagen guardada: {img_path}") + except requests.RequestException as e: + print(f"Error al descargar la imagen para {platform}/{name}: {e}") + create_empty_image(name, platform, save_folder, type) def fetch_image_data(game): name = game['name'] @@ -48,21 +43,17 @@ def fetch_image_data(game): url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}&hash={hash}" try: - response = requests.get(url) - if response.status_code == 200: - data = response.json() - img_url = data.get('img') # Usar get para evitar errores si 'img' no existe o es None - if img_url: - create_empty_image(name, platform, save_folder, type) - download_image(name, platform, img_url, save_folder, type) - else: - print(f"No se encontró una URL de imagen válida para {platform}/{name}. Creando archivo vacío.") - create_empty_image(name, platform, save_folder, type) + response = requests.get(url, timeout=10) + response.raise_for_status() # Lanza una excepción para códigos de error HTTP + data = response.json() + img_url = data.get('img') + if img_url: + download_image(name, platform, img_url, save_folder, type) else: - print(f"Error al llamar al servicio para {platform}/{name}") + print(f"No se encontró URL para {platform}/{name}. Creando archivo vacío.") create_empty_image(name, platform, save_folder, type) except requests.RequestException as e: - print(f"Excepción al procesar {platform}/{name}: {e}") + print(f"Error procesando {platform}/{name}: {e}") create_empty_image(name, platform, save_folder, type) def process_json(save_folder): @@ -74,6 +65,10 @@ def process_json(save_folder): with ThreadPoolExecutor(max_workers=5) as executor: futures = [executor.submit(fetch_image_data, game) for game in games] for future in as_completed(futures): - future.result() # Esperar a que cada tarea termine + try: + future.result() # Procesar excepciones lanzadas en cada tarea + except Exception as e: + print(f"Error en una tarea: {e}") -process_json(save_folder) +if __name__ == "__main__": + process_json(save_folder) From 7230314b8eebbb66db035eeef5507a99be244cf6 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 22 Nov 2024 15:19:34 +0100 Subject: [PATCH 104/230] type --- tools/retro-library/download_art.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 6593e935c..76a492570 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -40,7 +40,7 @@ def fetch_image_data(game): platform = game['platform'] hash = game['hash'] type = game['type'] - url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}&hash={hash}" + url = f"https://bot.emudeck.com/steamdbimg.php?name={name}&platform={platform}&hash={hash}&type={type}" try: response = requests.get(url, timeout=10) From 413c2f7bbab73686c01ca0bb89a493e574dcde7b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 12:18:48 +0100 Subject: [PATCH 105/230] $MSG --- functions/generateGameLists.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 85c1a595d..faa9c70f8 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -1,8 +1,11 @@ #!/bin/bash + +MSG="$HOME/.config/EmuDeck/msg.log" + generateGameLists() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" - + echo "Starting to build database" > "$MSG" mkdir -p "$storagePath/retrolibrary/artwork" mkdir -p "$accountfolder/config/grid/retrolibrary/" ln -s "$storagePath/retrolibrary/artwork/" "$accountfolder/config/grid/retrolibrary/artwork" @@ -13,11 +16,14 @@ generateGameLists() { pegasus_setPaths rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$dest_folder" --keep-dirlinks mkdir -p "$HOME/emudeck/cache/" - python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" + echo "Database built" > "$MSG" + #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" } generateGameListsJson() { + echo "Adding Games" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" + echo "Games Added" > "$MSG" #cat $HOME/emudeck/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & generateGameLists_artwork &> /dev/null & @@ -32,11 +38,13 @@ generateGameLists_artwork() { mkdir -p "$HOME/emudeck/cache/" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" + echo "Searching for missing artwork" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" + echo "Artwork finished. Restart if you see this message" > "$MSG" } saveImage(){ @@ -131,18 +139,25 @@ generateGameLists_retroAchievements(){ generateGameLists_downloadAchievements(){ local folder="$storagePath/retrolibrary/achievements" if [ ! -d $folder ]; then + echo "Downloading Retroachievements Data" > "$MSG" mkdir -p $folder wget -q -O "$folder/achievements.zip" "https://bot.emudeck.com/achievements/achievements.zip" cd "$folder" && unzip -o achievements.zip && rm achievements.zip + echo "Retroachievements Data Downloaded" > "$MSG" fi } generateGameLists_downloadData(){ local folder="$storagePath/retrolibrary/data" if [ ! -d $folder ]; then + echo "Downloading Metada" > "$MSG" mkdir -p $folder wget -q -O "$folder/data.zip" "https://bot.emudeck.com/data/data.zip" cd $folder && unzip -o data.zip && rm data.zip + echo "Metada Downloaded" > "$MSG" fi } +generateGameLists_readMessage(){ + cat "$MSG" +} From c964ca9b8104f72bbf81652dfa78df873dd372f7 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 12:26:56 +0100 Subject: [PATCH 106/230] python logs --- tools/retro-library/download_art.py | 50 ++++++--- tools/retro-library/download_art_platforms.py | 65 +++++++---- tools/retro-library/generate_game_lists.py | 91 ++++++--------- tools/retro-library/missing_artwork.py | 105 +++++++----------- .../missing_artwork_platforms.py | 15 +++ 5 files changed, 168 insertions(+), 158 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 76a492570..2577bef2a 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -4,35 +4,44 @@ import sys from concurrent.futures import ThreadPoolExecutor, as_completed -# Path del JSON y carpeta de destino desde los argumentos +# Path for the JSON and target folder from command-line arguments save_folder = sys.argv[1] json_path = os.path.expanduser('~/emudeck/cache/missing_artwork.json') +# Path for the log file +home_dir = os.environ.get("HOME") +msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") + +# Function to write messages to the log file +def log_message(message): + with open(msg_file, "a") as log_file: # "a" to append messages without overwriting + log_file.write(message + "\n") + def create_empty_image(name, platform, save_folder, type): extension = "jpg" if type != "wheel" else "png" - # Crear la ruta completa para la carpeta folder_path = os.path.join(save_folder, platform, "media", type) os.makedirs(folder_path, exist_ok=True) img_path = os.path.join(folder_path, f"{name}.{extension}") - # Crear archivo vacío with open(img_path, 'wb') as file: pass - print(f"Archivo vacío creado: {img_path}") + log_message(f"Empty file created: {img_path}") + print(f"Empty file created: {img_path}") def download_image(name, platform, img_url, save_folder, type): extension = "jpg" if type != "wheel" else "png" - # Crear la ruta completa para la carpeta folder_path = os.path.join(save_folder, platform, "media", type) os.makedirs(folder_path, exist_ok=True) img_path = os.path.join(folder_path, f"{name}.{extension}") try: response = requests.get(img_url, timeout=10) - response.raise_for_status() # Lanza una excepción para códigos de error HTTP + response.raise_for_status() # Raise an exception for HTTP error codes with open(img_path, 'wb') as file: file.write(response.content) - print(f"Imagen guardada: {img_path}") + log_message(f"Image saved: {img_path}") + print(f"Image saved: {img_path}") except requests.RequestException as e: - print(f"Error al descargar la imagen para {platform}/{name}: {e}") + log_message(f"Error downloading image for {platform}/{name}: {e}") + print(f"Error downloading image for {platform}/{name}: {e}") create_empty_image(name, platform, save_folder, type) def fetch_image_data(game): @@ -44,31 +53,42 @@ def fetch_image_data(game): try: response = requests.get(url, timeout=10) - response.raise_for_status() # Lanza una excepción para códigos de error HTTP + response.raise_for_status() # Raise an exception for HTTP error codes data = response.json() img_url = data.get('img') if img_url: download_image(name, platform, img_url, save_folder, type) else: - print(f"No se encontró URL para {platform}/{name}. Creando archivo vacío.") + log_message(f"No URL found for {platform}/{name}. Creating empty file.") + print(f"No URL found for {platform}/{name}. Creating empty file.") create_empty_image(name, platform, save_folder, type) except requests.RequestException as e: - print(f"Error procesando {platform}/{name}: {e}") + log_message(f"Error processing {platform}/{name}: {e}") + print(f"Error processing {platform}/{name}: {e}") create_empty_image(name, platform, save_folder, type) def process_json(save_folder): - # Leer el JSON original + # Read the original JSON with open(json_path, 'r') as file: games = json.load(file) - # Usar ThreadPoolExecutor para descargar en paralelo + log_message(f"Starting processing for {len(games)} games...") + print(f"Starting processing for {len(games)} games...") + + # Use ThreadPoolExecutor for parallel downloading with ThreadPoolExecutor(max_workers=5) as executor: futures = [executor.submit(fetch_image_data, game) for game in games] for future in as_completed(futures): try: - future.result() # Procesar excepciones lanzadas en cada tarea + future.result() # Process exceptions raised in tasks except Exception as e: - print(f"Error en una tarea: {e}") + log_message(f"Error in a task: {e}") + print(f"Error in a task: {e}") + + log_message("Processing completed.") + print("Processing completed.") if __name__ == "__main__": + log_message("Starting image processing script...") process_json(save_folder) + log_message("Image processing script completed.") diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 1a09683c7..c5a3b7869 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -5,63 +5,84 @@ from io import BytesIO import sys +# Define the log file path +home_dir = os.environ.get("HOME") +msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") + +# Function to write messages to the log file +def log_message(message): + with open(msg_file, "w") as log_file: # "a" to append messages without overwriting + log_file.write(message + "\n") + def download_and_extract(output_dir): - # Ruta fija del archivo JSON + # Fixed path to the JSON file json_file_path = os.path.expanduser("~/emudeck/cache/missing_systems.json") - # Verificar que el archivo JSON exista + # Check if the JSON file exists if not os.path.exists(json_file_path): - print(f"Archivo JSON no encontrado: {json_file_path}") + log_message(f"JSON file not found: {json_file_path}") + print(f"JSON file not found: {json_file_path}") return - # Leer el JSON + # Read the JSON with open(json_file_path, 'r') as f: try: data = json.load(f) except json.JSONDecodeError as e: - print(f"Error al leer el archivo JSON: {e}") + log_message(f"Error reading JSON file: {e}") + print(f"Error reading JSON file: {e}") return - # Verificar que el JSON sea una lista + # Verify that the JSON contains a list if not isinstance(data, list): - print("El archivo JSON no contiene una lista válida.") + log_message("The JSON file does not contain a valid list.") + print("The JSON file does not contain a valid list.") return if not data: - print("No hay plataformas en el archivo JSON.") + log_message("No platforms found in the JSON file.") + print("No platforms found in the JSON file.") return - # Crear el directorio de salida si no existe + # Create the output directory if it doesn't exist os.makedirs(output_dir, exist_ok=True) - # Procesar cada plataforma + # Process each platform for platform in data: url = f"https://bot.emudeck.com/artwork_deck/{platform}.zip" - print(f"Descargando: {url}") + log_message(f"Downloading: {url}") + print(f"Downloading: {url}") try: - # Descargar el archivo ZIP + # Download the ZIP file response = requests.get(url, stream=True) - response.raise_for_status() # Lanza un error si la descarga falla + response.raise_for_status() # Raise an error if the download fails - # Leer el contenido del ZIP en memoria + # Read the ZIP content in memory with zipfile.ZipFile(BytesIO(response.content)) as zip_file: - print(f"Extrayendo contenido de {platform}.zip a {output_dir}") - zip_file.extractall(output_dir) # Sobrescribe por defecto + log_message(f"Extracting content from {platform}.zip to {output_dir}") + print(f"Extracting content from {platform}.zip to {output_dir}") + zip_file.extractall(output_dir) # Overwrite by default except requests.exceptions.RequestException as e: - print(f"Error al descargar {url}: {e}") + log_message(f"Error downloading {url}: {e}") + print(f"Error downloading {url}: {e}") except zipfile.BadZipFile as e: - print(f"Error al procesar el ZIP para {platform}: {e}") + log_message(f"Error processing the ZIP file for {platform}: {e}") + print(f"Error processing the ZIP file for {platform}: {e}") - print("Proceso completado.") + log_message("Process completed.") + print("Process completed.") -# Verificar argumentos de línea de comandos +# Verify command-line arguments if len(sys.argv) != 2: - print("Uso: python3 download_and_extract.py ") + log_message("Incorrect usage: python3 download_and_extract.py ") + print("Usage: python3 download_and_extract.py ") sys.exit(1) -# Directorio de salida pasado como argumento +# Output directory passed as an argument output_dir = sys.argv[1] +log_message("Starting download and extraction process...") download_and_extract(output_dir) +log_message("Download and extraction process completed.") diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index f7d693ddd..e54763a7b 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -4,9 +4,18 @@ import re import hashlib +# Define the log file path +home_dir = os.environ.get("HOME") +msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") + +# Function to write messages to the log file +def log_message(message): + with open(msg_file, "a") as log_file: # "a" to append messages without overwriting + log_file.write(message + "\n") + def generate_game_lists(roms_path): def calculate_hash(file_path): - """Calcula el hash MD5 de un archivo.""" + """Calculate the MD5 hash of a file.""" hash_md5 = hashlib.md5() try: with open(file_path, "rb") as f: @@ -14,7 +23,7 @@ def calculate_hash(file_path): hash_md5.update(chunk) return hash_md5.hexdigest() except Exception as e: - print(f"Error al calcular el hash para {file_path}: {e}") + log_message(f"Error calculating hash for {file_path}: {e}") return None def collect_game_data(system_dir, extensions): @@ -29,73 +38,40 @@ def collect_game_data(system_dir, extensions): extension = filename.split('.')[-1] name = '.'.join(filename.split('.')[:-1]) if extension in extensions: + # Special cases for WiiU and PS3 if "wiiu" in system_dir: - if extension == "wux": - name = name - elif extension == "wua": - name = name - else: - parts = root.split(os.sep) - if len(parts) >= 2: - name = parts[-2] - else: - name = name - - if name == "roms": - name = name - - if name == "wiiu": - name = parts[-1] - + parts = root.split(os.sep) + name = parts[-2] if len(parts) >= 2 else name if "ps3" in system_dir: - parts = root.split(os.sep) - if len(parts) >= 3: - name = parts[-3] - else: - name = name - - if name == "roms": - name = name - - if name == "ps3": - name = parts[-1] + parts = root.split(os.sep) + name = parts[-3] if len(parts) >= 3 else name platform = os.path.basename(system_dir) + # Clean the game name name_cleaned = re.sub(r'\(.*?\)', '', name) name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) - name_cleaned = name_cleaned.strip() - - name_cleaned = name_cleaned.replace(' ', '_') - name_cleaned = name_cleaned.replace('-', '_') + name_cleaned = name_cleaned.strip().replace(' ', '_').replace('-', '_') name_cleaned = re.sub(r'_+', '_', name_cleaned) - name_cleaned = name_cleaned.replace('+', '') - name_cleaned = name_cleaned.replace('&', '') - name_cleaned = name_cleaned.replace('!', '') - name_cleaned = name_cleaned.replace("'", '') - name_cleaned = name_cleaned.replace('.', '') + name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') name_cleaned_pegasus = name.replace(',_', ',') - # Calcular el hash de la ROM + # Calculate the ROM hash rom_hash = calculate_hash(file_path) - clean_name = name_cleaned - game_img = f"/customimages/retrolibrary/artwork/{platform}/media/box2dfront/{clean_name}.jpg" - game_ss = f"/customimages/retrolibrary/artwork/{platform}/media/screenshot/{clean_name}.jpg" - game_wheel = f"/customimages/retrolibrary/artwork/{platform}/media/wheel/{clean_name}.png" game_info = { - "name": clean_name, + "name": name_cleaned, "filename": file_path, - "file": clean_name, - "img": game_img, - "ss": game_ss, - "wheel": game_wheel, + "file": name_cleaned, + "img": f"/customimages/retrolibrary/artwork/{platform}/media/box2dfront/{name_cleaned}.jpg", + "ss": f"/customimages/retrolibrary/artwork/{platform}/media/screenshot/{name_cleaned}.jpg", + "wheel": f"/customimages/retrolibrary/artwork/{platform}/media/wheel/{name_cleaned}.png", "platform": platform, - "hash": rom_hash # Agregar el hash + "hash": rom_hash } game_data.append(game_info) - game_data_sorted = sorted(game_data, key=lambda x: x['name']) - return game_data_sorted + log_message(f"Game added: {game_info}") + return sorted(game_data, key=lambda x: x['name']) roms_dir = roms_path valid_system_dirs = [] @@ -103,18 +79,18 @@ def collect_game_data(system_dir, extensions): for system_dir in os.listdir(roms_dir): if system_dir == "xbox360": system_dir = "xbox360/roms" - #if system_dir == "wiiu": - # system_dir = "wiiu/roms" full_path = os.path.join(roms_dir, system_dir) if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) if file_count > 2: valid_system_dirs.append(full_path) + log_message(f"Valid system directory found: {full_path}") game_list = [] for system_dir in valid_system_dirs: if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): + log_message(f"Skipping directory: {system_dir}") continue with open(os.path.join(system_dir, 'metadata.txt')) as f: @@ -133,16 +109,19 @@ def collect_game_data(system_dir, extensions): "games": games } game_list.append(system_info) - game_list_sorted = sorted(game_list, key=lambda x: x['title']) + log_message(f"Collected {len(games)} games from {system_dir}") - json_output = json.dumps(game_list_sorted, indent=4) + json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) home_directory = os.path.expanduser("~") output_file = os.path.join(home_directory, 'emudeck', 'cache', 'roms_games.json') os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) + log_message(f"JSON output saved to: {output_file}") print(json_output) roms_path = sys.argv[1] +log_message("Starting game list generation...") generate_game_lists(f"{roms_path}") +log_message("Game list generation completed.") diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index ae6426772..5981a2c8b 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -4,6 +4,15 @@ import re import hashlib +# Define the log file path +home_dir = os.environ.get("HOME") +msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") + +# Function to write messages to the log file +def log_message(message): + with open(msg_file, "w") as log_file: # "a" to append messages without overwriting + log_file.write(message + "\n") + def generate_game_lists(roms_path, images_path): def calculate_hash(file_path): hash_md5 = hashlib.md5() @@ -13,8 +22,9 @@ def calculate_hash(file_path): hash_md5.update(chunk) return hash_md5.hexdigest() except Exception as e: - #print(f"Error al calcular el hash para {file_path}: {e}") + log_message(f"Error calculating hash for {file_path}: {e}") return None + def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -27,82 +37,38 @@ def collect_game_data(system_dir, extensions): extension = filename.split('.')[-1] name = '.'.join(filename.split('.')[:-1]) if extension in extensions: + # Custom logic for specific systems if "wiiu" in system_dir: - if extension == "wux": - name = name - elif extension == "wua": - name = name - else: - parts = root.split(os.sep) - if len(parts) >= 2: - name = parts[-2] - else: - name = name - - if name == "roms": - name = name - - if name == "wiiu": - name = parts[-1] + parts = root.split(os.sep) + name = parts[-2] if len(parts) >= 2 else name if "ps3" in system_dir: - parts = root.split(os.sep) - if len(parts) >= 3: - name = parts[-3] - else: - name = name - - if name == "roms": - name = name - - if name == "ps3": - name = parts[-1] + parts = root.split(os.sep) + name = parts[-3] if len(parts) >= 3 else name platform = os.path.basename(system_dir) + # Clean the game name name_cleaned = re.sub(r'\(.*?\)', '', name) name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) - name_cleaned = name_cleaned.strip() - name_cleaned = name_cleaned.replace(' ', '_').replace('-', '_') + name_cleaned = name_cleaned.strip().replace(' ', '_').replace('-', '_') name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') rom_hash = calculate_hash(file_path) - # Verificar si la imagen existe en el images_path - - img_path = os.path.join(images_path, f"{platform}/media/box2dfront/{name_cleaned}.jpg") - - ##print(img_path) - if not os.path.exists(img_path): - game_info = { - "name": name_cleaned, - "platform": platform, - "hash": rom_hash, - "type": "box2dfront" - } - game_data.append(game_info) - - img_path = os.path.join(images_path, f"{platform}/media/wheel/{name_cleaned}.png") - #print(img_path) - if not os.path.exists(img_path): - game_info = { - "name": name_cleaned, - "platform": platform, - "hash": rom_hash, - "type": "wheel" - } - game_data.append(game_info) - img_path = os.path.join(images_path, f"{platform}/media/screenshot/{name_cleaned}.jpg") - ##print(img_path) - if not os.path.exists(img_path): - game_info = { - "name": name_cleaned, - "platform": platform, - "hash": rom_hash, - "type": "screenshot" - } - game_data.append(game_info) + # Check for missing images + for img_type, ext in [("box2dfront", ".jpg"), ("wheel", ".png"), ("screenshot", ".jpg")]: + img_path = os.path.join(images_path, f"{platform}/media/{img_type}/{name_cleaned}{ext}") + if not os.path.exists(img_path): + game_info = { + "name": name_cleaned, + "platform": platform, + "hash": rom_hash, + "type": img_type + } + game_data.append(game_info) + log_message(f"Missing {img_type} image: {img_path}") game_data_sorted = sorted(game_data, key=lambda x: x['name']) return game_data_sorted @@ -110,6 +76,7 @@ def collect_game_data(system_dir, extensions): roms_dir = roms_path valid_system_dirs = [] + # Validate system directories for system_dir in os.listdir(roms_dir): if system_dir == "xbox360": system_dir = "xbox360/roms" @@ -118,11 +85,14 @@ def collect_game_data(system_dir, extensions): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) if file_count > 2: valid_system_dirs.append(full_path) + log_message(f"Valid system directory found: {full_path}") game_list = [] + # Process each system directory for system_dir in valid_system_dirs: if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): + log_message(f"Skipping directory: {system_dir}") continue with open(os.path.join(system_dir, 'metadata.txt')) as f: @@ -132,16 +102,21 @@ def collect_game_data(system_dir, extensions): games = collect_game_data(system_dir, extensions) if games: game_list.extend(games) + log_message(f"Collected {len(games)} games from {system_dir}") + # Save the JSON output json_output = json.dumps(game_list, indent=4) home_directory = os.path.expanduser("~") output_file = os.path.join(home_directory, 'emudeck', 'cache', 'missing_artwork.json') os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) + log_message(f"JSON output saved to: {output_file}") -# Pasar la ruta de las ROMs y de las imágenes desde los argumentos de línea de comandos +# Read ROMs and images paths from command-line arguments roms_path = sys.argv[1] images_path = sys.argv[2] +log_message("Starting game list generation process...") generate_game_lists(roms_path, images_path) +log_message("Game list generation process completed.") diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 7ecbdc721..6edde1196 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -3,6 +3,14 @@ import sys import re +home_dir = os.environ.get("HOME") +msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") + +# Función para escribir en el archivo de log +def log_message(message): + with open(msg_file, "w") as log_file: # "a" para agregar mensajes sin sobrescribir + log_file.write(message + "\n") + def generate_systems_with_missing_images(roms_path, images_path): def has_missing_images(system_dir, extensions): for root, _, files in os.walk(system_dir): @@ -26,6 +34,7 @@ def has_missing_images(system_dir, extensions): img_path = os.path.join(images_path, f"{platform}/{name_cleaned}.jpg") if not os.path.exists(img_path): + log_message(f"Missing image: {img_path}") return True return False @@ -40,11 +49,13 @@ def has_missing_images(system_dir, extensions): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) if file_count > 2: valid_system_dirs.append(full_path) + log_message(f"Valid system directory added: {full_path}") systems_with_missing_images = set() for system_dir in valid_system_dirs: if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): + log_message(f"Skipping directory: {system_dir}") continue with open(os.path.join(system_dir, 'metadata.txt')) as f: @@ -53,6 +64,7 @@ def has_missing_images(system_dir, extensions): if has_missing_images(system_dir, extensions): systems_with_missing_images.add(os.path.basename(system_dir)) + log_message(f"System with missing images: {os.path.basename(system_dir)}") json_output = json.dumps(list(systems_with_missing_images), indent=4) home_directory = os.path.expanduser("~") @@ -60,9 +72,12 @@ def has_missing_images(system_dir, extensions): os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) + log_message(f"JSON output saved to: {output_file}") # Pasar la ruta de las ROMs y de las imágenes desde los argumentos de línea de comandos roms_path = sys.argv[1] images_path = sys.argv[2] +log_message("Searching missing artwork in bundles...") generate_systems_with_missing_images(roms_path, images_path) +log_message("Completed missing artwork in bundles") From 7d3ad045cae7a2a93e3e0e105308c23eb0a5f246 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 12:50:13 +0100 Subject: [PATCH 107/230] speeding things up --- functions/generateGameLists.sh | 6 ++---- tools/retro-library/download_art_platforms.py | 13 +++++-------- tools/retro-library/generate_game_lists.py | 5 +---- tools/retro-library/missing_artwork.py | 9 +++------ tools/retro-library/missing_artwork_platforms.py | 2 -- 5 files changed, 11 insertions(+), 24 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index faa9c70f8..4cdc39068 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -39,11 +39,9 @@ generateGameLists_artwork() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Searching for missing artwork" > "$MSG" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" - python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" - python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" & echo "Artwork finished. Restart if you see this message" > "$MSG" } diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index c5a3b7869..693210c52 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -50,8 +50,8 @@ def download_and_extract(output_dir): # Process each platform for platform in data: url = f"https://bot.emudeck.com/artwork_deck/{platform}.zip" - log_message(f"Downloading: {url}") - print(f"Downloading: {url}") + log_message(f"Downloading: {platform}") + print(f"Downloading: {platform}") try: # Download the ZIP file @@ -60,29 +60,26 @@ def download_and_extract(output_dir): # Read the ZIP content in memory with zipfile.ZipFile(BytesIO(response.content)) as zip_file: - log_message(f"Extracting content from {platform}.zip to {output_dir}") print(f"Extracting content from {platform}.zip to {output_dir}") zip_file.extractall(output_dir) # Overwrite by default except requests.exceptions.RequestException as e: - log_message(f"Error downloading {url}: {e}") + log_message(f"Error downloading {platform}: {e}") print(f"Error downloading {url}: {e}") except zipfile.BadZipFile as e: log_message(f"Error processing the ZIP file for {platform}: {e}") print(f"Error processing the ZIP file for {platform}: {e}") - log_message("Process completed.") print("Process completed.") # Verify command-line arguments if len(sys.argv) != 2: - log_message("Incorrect usage: python3 download_and_extract.py ") print("Usage: python3 download_and_extract.py ") sys.exit(1) # Output directory passed as an argument output_dir = sys.argv[1] -log_message("Starting download and extraction process...") +log_message("Starting download and extraction process for bundles...") download_and_extract(output_dir) -log_message("Download and extraction process completed.") +log_message("Download and extraction process completed for bundles") diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index e54763a7b..9a1f62c5a 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -23,7 +23,6 @@ def calculate_hash(file_path): hash_md5.update(chunk) return hash_md5.hexdigest() except Exception as e: - log_message(f"Error calculating hash for {file_path}: {e}") return None def collect_game_data(system_dir, extensions): @@ -70,7 +69,6 @@ def collect_game_data(system_dir, extensions): "hash": rom_hash } game_data.append(game_info) - log_message(f"Game added: {game_info}") return sorted(game_data, key=lambda x: x['name']) roms_dir = roms_path @@ -109,7 +107,7 @@ def collect_game_data(system_dir, extensions): "games": games } game_list.append(system_info) - log_message(f"Collected {len(games)} games from {system_dir}") + log_message(f"Detected {len(games)} games from {system_dir}") json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) home_directory = os.path.expanduser("~") @@ -117,7 +115,6 @@ def collect_game_data(system_dir, extensions): os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) - log_message(f"JSON output saved to: {output_file}") print(json_output) roms_path = sys.argv[1] diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 5981a2c8b..2d8bf5d76 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -22,7 +22,6 @@ def calculate_hash(file_path): hash_md5.update(chunk) return hash_md5.hexdigest() except Exception as e: - log_message(f"Error calculating hash for {file_path}: {e}") return None def collect_game_data(system_dir, extensions): @@ -68,7 +67,6 @@ def collect_game_data(system_dir, extensions): "type": img_type } game_data.append(game_info) - log_message(f"Missing {img_type} image: {img_path}") game_data_sorted = sorted(game_data, key=lambda x: x['name']) return game_data_sorted @@ -102,7 +100,7 @@ def collect_game_data(system_dir, extensions): games = collect_game_data(system_dir, extensions) if games: game_list.extend(games) - log_message(f"Collected {len(games)} games from {system_dir}") + log_message(f"Found {len(games)} missing artwork from {system_dir}") # Save the JSON output json_output = json.dumps(game_list, indent=4) @@ -111,12 +109,11 @@ def collect_game_data(system_dir, extensions): os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) - log_message(f"JSON output saved to: {output_file}") # Read ROMs and images paths from command-line arguments roms_path = sys.argv[1] images_path = sys.argv[2] -log_message("Starting game list generation process...") +log_message("Missing artwork list generation in process...") generate_game_lists(roms_path, images_path) -log_message("Game list generation process completed.") +log_message("Missing artwork list process completed.") diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 6edde1196..e54e1adb3 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -34,7 +34,6 @@ def has_missing_images(system_dir, extensions): img_path = os.path.join(images_path, f"{platform}/{name_cleaned}.jpg") if not os.path.exists(img_path): - log_message(f"Missing image: {img_path}") return True return False @@ -72,7 +71,6 @@ def has_missing_images(system_dir, extensions): os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) - log_message(f"JSON output saved to: {output_file}") # Pasar la ruta de las ROMs y de las imágenes desde los argumentos de línea de comandos roms_path = sys.argv[1] From e2ea91f1ae5b430cd3d15d8826079567726da3d7 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 12:54:36 +0100 Subject: [PATCH 108/230] detailled log --- tools/retro-library/generate_game_lists.py | 10 +++++----- tools/retro-library/missing_artwork.py | 10 +++++----- tools/retro-library/missing_artwork_platforms.py | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 9a1f62c5a..a39e11b26 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -82,13 +82,13 @@ def collect_game_data(system_dir, extensions): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) if file_count > 2: valid_system_dirs.append(full_path) - log_message(f"Valid system directory found: {full_path}") + log_message(f"GGL: Valid system directory found: {full_path}") game_list = [] for system_dir in valid_system_dirs: if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): - log_message(f"Skipping directory: {system_dir}") + log_message(f"GGL: Skipping directory: {system_dir}") continue with open(os.path.join(system_dir, 'metadata.txt')) as f: @@ -107,7 +107,7 @@ def collect_game_data(system_dir, extensions): "games": games } game_list.append(system_info) - log_message(f"Detected {len(games)} games from {system_dir}") + log_message(f"GGL: Detected {len(games)} games from {system_dir}") json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) home_directory = os.path.expanduser("~") @@ -119,6 +119,6 @@ def collect_game_data(system_dir, extensions): roms_path = sys.argv[1] -log_message("Starting game list generation...") +log_message("GGL: Starting game list generation...") generate_game_lists(f"{roms_path}") -log_message("Game list generation completed.") +log_message("GGL: Game list generation completed.") diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 2d8bf5d76..d4d3c825c 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -83,14 +83,14 @@ def collect_game_data(system_dir, extensions): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) if file_count > 2: valid_system_dirs.append(full_path) - log_message(f"Valid system directory found: {full_path}") + log_message(f"MA: Valid system directory found: {full_path}") game_list = [] # Process each system directory for system_dir in valid_system_dirs: if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): - log_message(f"Skipping directory: {system_dir}") + log_message(f"MA: Skipping directory: {system_dir}") continue with open(os.path.join(system_dir, 'metadata.txt')) as f: @@ -100,7 +100,7 @@ def collect_game_data(system_dir, extensions): games = collect_game_data(system_dir, extensions) if games: game_list.extend(games) - log_message(f"Found {len(games)} missing artwork from {system_dir}") + log_message(f"MA: Found {len(games)} missing artwork from {system_dir}") # Save the JSON output json_output = json.dumps(game_list, indent=4) @@ -114,6 +114,6 @@ def collect_game_data(system_dir, extensions): roms_path = sys.argv[1] images_path = sys.argv[2] -log_message("Missing artwork list generation in process...") +log_message("MA: Missing artwork list generation in process...") generate_game_lists(roms_path, images_path) -log_message("Missing artwork list process completed.") +log_message("MA: Missing artwork list process completed.") diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index e54e1adb3..b43bd5626 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -48,13 +48,13 @@ def has_missing_images(system_dir, extensions): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) if file_count > 2: valid_system_dirs.append(full_path) - log_message(f"Valid system directory added: {full_path}") + log_message(f"MAP: Valid system directory added: {full_path}") systems_with_missing_images = set() for system_dir in valid_system_dirs: if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): - log_message(f"Skipping directory: {system_dir}") + log_message(f"MAP: Skipping directory: {system_dir}") continue with open(os.path.join(system_dir, 'metadata.txt')) as f: @@ -63,7 +63,7 @@ def has_missing_images(system_dir, extensions): if has_missing_images(system_dir, extensions): systems_with_missing_images.add(os.path.basename(system_dir)) - log_message(f"System with missing images: {os.path.basename(system_dir)}") + log_message(f"MAP: System with missing images: {os.path.basename(system_dir)}") json_output = json.dumps(list(systems_with_missing_images), indent=4) home_directory = os.path.expanduser("~") @@ -76,6 +76,6 @@ def has_missing_images(system_dir, extensions): roms_path = sys.argv[1] images_path = sys.argv[2] -log_message("Searching missing artwork in bundles...") +log_message("MAP: Searching missing artwork in bundles...") generate_systems_with_missing_images(roms_path, images_path) -log_message("Completed missing artwork in bundles") +log_message("MAP: Completed missing artwork in bundles") From 47feee5fe755898858e541d93368206312d7be36 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 13:08:47 +0100 Subject: [PATCH 109/230] download bundle if not exists --- tools/retro-library/download_art_platforms.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 693210c52..231f49d1f 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -44,11 +44,14 @@ def download_and_extract(output_dir): print("No platforms found in the JSON file.") return - # Create the output directory if it doesn't exist - os.makedirs(output_dir, exist_ok=True) - # Process each platform for platform in data: + extracted_folder = os.path.join(output_dir, platform) + if os.path.exists(extracted_folder): + log_message(f"Skipped: {platform} already extracted at {extracted_folder}.") + print(f"Skipped: {platform} already extracted at {extracted_folder}.") + continue + url = f"https://bot.emudeck.com/artwork_deck/{platform}.zip" log_message(f"Downloading: {platform}") print(f"Downloading: {platform}") @@ -63,6 +66,9 @@ def download_and_extract(output_dir): print(f"Extracting content from {platform}.zip to {output_dir}") zip_file.extractall(output_dir) # Overwrite by default + log_message(f"Extracted: {platform} to {output_dir}") + print(f"Extracted: {platform} to {output_dir}") + except requests.exceptions.RequestException as e: log_message(f"Error downloading {platform}: {e}") print(f"Error downloading {url}: {e}") @@ -80,6 +86,9 @@ def download_and_extract(output_dir): # Output directory passed as an argument output_dir = sys.argv[1] +# Create the output directory if it doesn't exist +os.makedirs(output_dir, exist_ok=True) + log_message("Starting download and extraction process for bundles...") download_and_extract(output_dir) -log_message("Download and extraction process completed for bundles") +log_message("Download and extraction process completed for bundles.") From 2bf154b67acd777755cf55d199e521b0c5afc759 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 13:48:20 +0100 Subject: [PATCH 110/230] bg artwork --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 4cdc39068..b7d6656ca 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -41,7 +41,7 @@ generateGameLists_artwork() { echo "Searching for missing artwork" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder" & + $(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & echo "Artwork finished. Restart if you see this message" > "$MSG" } From 25b05bc717c21b5f287dc929e009329df3b93d0c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 13:52:45 +0100 Subject: [PATCH 111/230] temp no art --- functions/generateGameLists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index b7d6656ca..72bc14290 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -39,9 +39,9 @@ generateGameLists_artwork() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Searching for missing artwork" > "$MSG" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" + #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" - $(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & + #$(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & echo "Artwork finished. Restart if you see this message" > "$MSG" } From a9697c29d417392f6222f60a07716eede20ead29 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 14:32:58 +0100 Subject: [PATCH 112/230] clean json --- functions/generateGameLists.sh | 4 ++-- tools/retro-library/generate_game_lists.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 72bc14290..b7d6656ca 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -39,9 +39,9 @@ generateGameLists_artwork() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Searching for missing artwork" > "$MSG" - #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" - #$(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & + $(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & echo "Artwork finished. Restart if you see this message" > "$MSG" } diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index a39e11b26..2935f9227 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -61,10 +61,6 @@ def collect_game_data(system_dir, extensions): game_info = { "name": name_cleaned, "filename": file_path, - "file": name_cleaned, - "img": f"/customimages/retrolibrary/artwork/{platform}/media/box2dfront/{name_cleaned}.jpg", - "ss": f"/customimages/retrolibrary/artwork/{platform}/media/screenshot/{name_cleaned}.jpg", - "wheel": f"/customimages/retrolibrary/artwork/{platform}/media/wheel/{name_cleaned}.png", "platform": platform, "hash": rom_hash } From 5128b69a7bfa7c60704de6c727fb62b5d87aae8a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 14:42:01 +0100 Subject: [PATCH 113/230] cleanup --- tools/retro-library/generate_game_lists.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 2935f9227..80c3116d9 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -61,6 +61,8 @@ def collect_game_data(system_dir, extensions): game_info = { "name": name_cleaned, "filename": file_path, + "file": name_cleaned, + "img": f"/customimages/retrolibrary/artwork/{platform}/media", "platform": platform, "hash": rom_hash } From abb23222b896065747ebad9b578a022662597fcf Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 14:53:05 +0100 Subject: [PATCH 114/230] temp --- functions/generateGameLists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index b7d6656ca..72bc14290 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -39,9 +39,9 @@ generateGameLists_artwork() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Searching for missing artwork" > "$MSG" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" + #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" - $(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & + #$(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & echo "Artwork finished. Restart if you see this message" > "$MSG" } From bc6d11a6c7e68aab067e56eb61aa25bb05aa9355 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 17:06:40 +0100 Subject: [PATCH 115/230] testing --- functions/generateGameLists.sh | 7 +++---- tools/retro-library/generate_game_lists.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 72bc14290..2a84c101c 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -17,12 +17,12 @@ generateGameLists() { rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$dest_folder" --keep-dirlinks mkdir -p "$HOME/emudeck/cache/" echo "Database built" > "$MSG" - #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" + python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" } generateGameListsJson() { echo "Adding Games" > "$MSG" - python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" + #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" echo "Games Added" > "$MSG" #cat $HOME/emudeck/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & @@ -85,8 +85,7 @@ generateGameLists_getPercentage() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" - + #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" local json_file="$HOME/emudeck/cache/roms_games.json" local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 80c3116d9..1596a7e3d 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -113,7 +113,7 @@ def collect_game_data(system_dir, extensions): os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) - print(json_output) + #print(json_output) roms_path = sys.argv[1] From dbdfa4741cb75a7199275e08c5d5f730261dc1cb Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 17:43:26 +0100 Subject: [PATCH 116/230] artwork back --- functions/generateGameLists.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 2a84c101c..b636ce2ff 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -39,9 +39,9 @@ generateGameLists_artwork() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Searching for missing artwork" > "$MSG" - #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" - #$(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & + $(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & echo "Artwork finished. Restart if you see this message" > "$MSG" } @@ -85,7 +85,7 @@ generateGameLists_getPercentage() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" - #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" local json_file="$HOME/emudeck/cache/roms_games.json" local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" From e617985004dd13221273ee4c4935a814bb10c4fd Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 17:51:29 +0100 Subject: [PATCH 117/230] new cache paths py --- tools/retro-library/download_art.py | 32 ++++++++++++++++- tools/retro-library/download_art_platforms.py | 33 ++++++++++++++++- tools/retro-library/generate_game_lists.py | 36 +++++++++++++++++-- tools/retro-library/missing_artwork.py | 33 ++++++++++++++++- .../missing_artwork_platforms.py | 35 ++++++++++++++++-- 5 files changed, 162 insertions(+), 7 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 2577bef2a..23995c369 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -4,9 +4,39 @@ import sys from concurrent.futures import ThreadPoolExecutor, as_completed +async def getSettings(self): + pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') + user_home = os.path.expanduser("~") + if os.name == 'nt': + config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') + else: + config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} + + with open(config_file_path, 'r') as file: + for line in file: + match = pattern.search(line) + if match: + variable = match.group(1) + value = match.group(2).strip('"') + configuration[variable] = value + + if os.name == 'nt': + bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + else: + bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + configuration["branch"] = result.stdout.strip() + + configuration["systemOS"] = os.name + + return configuration + +settings = await getSettings() + # Path for the JSON and target folder from command-line arguments save_folder = sys.argv[1] -json_path = os.path.expanduser('~/emudeck/cache/missing_artwork.json') +json_path = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_artwork.json") # Path for the log file home_dir = os.environ.get("HOME") diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 231f49d1f..de31bc36c 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -9,6 +9,37 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") +async def getSettings(self): + pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') + user_home = os.path.expanduser("~") + if os.name == 'nt': + config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') + else: + config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} + + with open(config_file_path, 'r') as file: + for line in file: + match = pattern.search(line) + if match: + variable = match.group(1) + value = match.group(2).strip('"') + configuration[variable] = value + + if os.name == 'nt': + bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + else: + bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + configuration["branch"] = result.stdout.strip() + + configuration["systemOS"] = os.name + + return configuration + +settings = await getSettings() + + # Function to write messages to the log file def log_message(message): with open(msg_file, "w") as log_file: # "a" to append messages without overwriting @@ -16,7 +47,7 @@ def log_message(message): def download_and_extract(output_dir): # Fixed path to the JSON file - json_file_path = os.path.expanduser("~/emudeck/cache/missing_systems.json") + json_file_path = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_systems.json") # Check if the JSON file exists if not os.path.exists(json_file_path): diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 1596a7e3d..53dddebf0 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -8,6 +8,37 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") +async def getSettings(self): + pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') + user_home = os.path.expanduser("~") + if os.name == 'nt': + config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') + else: + config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} + + with open(config_file_path, 'r') as file: + for line in file: + match = pattern.search(line) + if match: + variable = match.group(1) + value = match.group(2).strip('"') + configuration[variable] = value + + if os.name == 'nt': + bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + else: + bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + configuration["branch"] = result.stdout.strip() + + configuration["systemOS"] = os.name + + return configuration + +settings = await getSettings() + + # Function to write messages to the log file def log_message(message): with open(msg_file, "a") as log_file: # "a" to append messages without overwriting @@ -108,8 +139,9 @@ def collect_game_data(system_dir, extensions): log_message(f"GGL: Detected {len(games)} games from {system_dir}") json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) - home_directory = os.path.expanduser("~") - output_file = os.path.join(home_directory, 'emudeck', 'cache', 'roms_games.json') + + output_file = os.path.join(settings["storagePath"], "/retrolibrary/cache/roms_games.json") + os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index d4d3c825c..4c9fae864 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -8,6 +8,37 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") +async def getSettings(self): + pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') + user_home = os.path.expanduser("~") + if os.name == 'nt': + config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') + else: + config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} + + with open(config_file_path, 'r') as file: + for line in file: + match = pattern.search(line) + if match: + variable = match.group(1) + value = match.group(2).strip('"') + configuration[variable] = value + + if os.name == 'nt': + bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + else: + bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + configuration["branch"] = result.stdout.strip() + + configuration["systemOS"] = os.name + + return configuration + +settings = await getSettings() + + # Function to write messages to the log file def log_message(message): with open(msg_file, "w") as log_file: # "a" to append messages without overwriting @@ -105,7 +136,7 @@ def collect_game_data(system_dir, extensions): # Save the JSON output json_output = json.dumps(game_list, indent=4) home_directory = os.path.expanduser("~") - output_file = os.path.join(home_directory, 'emudeck', 'cache', 'missing_artwork.json') + output_file = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_artwork.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index b43bd5626..48b719cf2 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -6,6 +6,37 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") +async def getSettings(self): + pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') + user_home = os.path.expanduser("~") + if os.name == 'nt': + config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') + else: + config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} + + with open(config_file_path, 'r') as file: + for line in file: + match = pattern.search(line) + if match: + variable = match.group(1) + value = match.group(2).strip('"') + configuration[variable] = value + + if os.name == 'nt': + bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + else: + bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + configuration["branch"] = result.stdout.strip() + + configuration["systemOS"] = os.name + + return configuration + +settings = await getSettings() + + # Función para escribir en el archivo de log def log_message(message): with open(msg_file, "w") as log_file: # "a" para agregar mensajes sin sobrescribir @@ -66,8 +97,8 @@ def has_missing_images(system_dir, extensions): log_message(f"MAP: System with missing images: {os.path.basename(system_dir)}") json_output = json.dumps(list(systems_with_missing_images), indent=4) - home_directory = os.path.expanduser("~") - output_file = os.path.join(home_directory, 'emudeck', 'cache', 'missing_systems.json') + + output_file = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_systems.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) From 1491e264adf51acccf244ad385a64284236c4978 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 17:54:37 +0100 Subject: [PATCH 118/230] noasync --- tools/retro-library/download_art.py | 4 ++-- tools/retro-library/download_art_platforms.py | 4 ++-- tools/retro-library/generate_game_lists.py | 4 ++-- tools/retro-library/missing_artwork.py | 4 ++-- tools/retro-library/missing_artwork_platforms.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 23995c369..817afa757 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -4,7 +4,7 @@ import sys from concurrent.futures import ThreadPoolExecutor, as_completed -async def getSettings(self): +def getSettings(self): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': @@ -32,7 +32,7 @@ async def getSettings(self): return configuration -settings = await getSettings() +settings = getSettings() # Path for the JSON and target folder from command-line arguments save_folder = sys.argv[1] diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index de31bc36c..701aa2edd 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -9,7 +9,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -async def getSettings(self): +def getSettings(self): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': @@ -37,7 +37,7 @@ async def getSettings(self): return configuration -settings = await getSettings() +settings = getSettings() # Function to write messages to the log file diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 53dddebf0..0c614f162 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -8,7 +8,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -async def getSettings(self): +def getSettings(self): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': @@ -36,7 +36,7 @@ async def getSettings(self): return configuration -settings = await getSettings() +settings = getSettings() # Function to write messages to the log file diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 4c9fae864..8cd5bf3d0 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -8,7 +8,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -async def getSettings(self): +def getSettings(self): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': @@ -36,7 +36,7 @@ async def getSettings(self): return configuration -settings = await getSettings() +settings = getSettings() # Function to write messages to the log file diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 48b719cf2..9e77b57c8 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -6,7 +6,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -async def getSettings(self): +def getSettings(self): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': @@ -34,7 +34,7 @@ async def getSettings(self): return configuration -settings = await getSettings() +settings = getSettings() # Función para escribir en el archivo de log From b34464223d0c790847c05bc63221b066199b6ce1 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 17:56:56 +0100 Subject: [PATCH 119/230] unself --- tools/retro-library/download_art.py | 2 +- tools/retro-library/download_art_platforms.py | 2 +- tools/retro-library/generate_game_lists.py | 2 +- tools/retro-library/missing_artwork.py | 2 +- tools/retro-library/missing_artwork_platforms.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 817afa757..5d0031f1c 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -4,7 +4,7 @@ import sys from concurrent.futures import ThreadPoolExecutor, as_completed -def getSettings(self): +def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 701aa2edd..0606f3595 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -9,7 +9,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -def getSettings(self): +def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 0c614f162..80f04132f 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -8,7 +8,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -def getSettings(self): +def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 8cd5bf3d0..628cee009 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -8,7 +8,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -def getSettings(self): +def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 9e77b57c8..a935cec5f 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -6,7 +6,7 @@ home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") -def getSettings(self): +def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") if os.name == 'nt': From 32802089cfda86323f01669d042ac4ad6ced5e2b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 17:57:54 +0100 Subject: [PATCH 120/230] import subprocess --- tools/retro-library/download_art.py | 1 + tools/retro-library/download_art_platforms.py | 1 + tools/retro-library/generate_game_lists.py | 1 + tools/retro-library/missing_artwork.py | 1 + tools/retro-library/missing_artwork_platforms.py | 1 + 5 files changed, 5 insertions(+) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 5d0031f1c..3f0e6bdc9 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -3,6 +3,7 @@ import os import sys from concurrent.futures import ThreadPoolExecutor, as_completed +import subprocess def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 0606f3595..e3eee0754 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -1,5 +1,6 @@ import os import json +import subprocess import requests import zipfile from io import BytesIO diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 80f04132f..75c4f3940 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -3,6 +3,7 @@ import sys import re import hashlib +import subprocess # Define the log file path home_dir = os.environ.get("HOME") diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 628cee009..6d444d3d7 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -3,6 +3,7 @@ import sys import re import hashlib +import subprocess # Define the log file path home_dir = os.environ.get("HOME") diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index a935cec5f..58a2f7d65 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -2,6 +2,7 @@ import json import sys import re +import subprocess home_dir = os.environ.get("HOME") msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") From f8362a34d4b07cfcf3619e156f57bf82b3aef166 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 18:01:40 +0100 Subject: [PATCH 121/230] fml --- tools/retro-library/download_art.py | 3 ++- tools/retro-library/download_art_platforms.py | 3 ++- tools/retro-library/generate_game_lists.py | 3 ++- tools/retro-library/missing_artwork.py | 3 ++- tools/retro-library/missing_artwork_platforms.py | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 3f0e6bdc9..c074b3ee4 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -34,10 +34,11 @@ def getSettings(): return configuration settings = getSettings() +storage_path = os.path.expandvars(settings["storagePath"]) # Path for the JSON and target folder from command-line arguments save_folder = sys.argv[1] -json_path = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_artwork.json") +json_path = os.path.join(storage_path, "/retrolibrary/cache/missing_artwork.json") # Path for the log file home_dir = os.environ.get("HOME") diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index e3eee0754..e43ae90df 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -39,6 +39,7 @@ def getSettings(): return configuration settings = getSettings() +storage_path = os.path.expandvars(settings["storagePath"]) # Function to write messages to the log file @@ -48,7 +49,7 @@ def log_message(message): def download_and_extract(output_dir): # Fixed path to the JSON file - json_file_path = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_systems.json") + json_file_path = os.path.join(storage_path, "/retrolibrary/cache/missing_systems.json") # Check if the JSON file exists if not os.path.exists(json_file_path): diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 75c4f3940..4b25d1d8a 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -38,6 +38,7 @@ def getSettings(): return configuration settings = getSettings() +storage_path = os.path.expandvars(settings["storagePath"]) # Function to write messages to the log file @@ -141,7 +142,7 @@ def collect_game_data(system_dir, extensions): json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) - output_file = os.path.join(settings["storagePath"], "/retrolibrary/cache/roms_games.json") + output_file = os.path.join(storage_path, "/retrolibrary/cache/roms_games.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 6d444d3d7..28a986598 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -38,6 +38,7 @@ def getSettings(): return configuration settings = getSettings() +storage_path = os.path.expandvars(settings["storagePath"]) # Function to write messages to the log file @@ -137,7 +138,7 @@ def collect_game_data(system_dir, extensions): # Save the JSON output json_output = json.dumps(game_list, indent=4) home_directory = os.path.expanduser("~") - output_file = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_artwork.json") + output_file = os.path.join(storage_path, "/retrolibrary/cache/missing_artwork.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 58a2f7d65..0cfb0a0d6 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -36,6 +36,7 @@ def getSettings(): return configuration settings = getSettings() +storage_path = os.path.expandvars(settings["storagePath"]) # Función para escribir en el archivo de log @@ -99,7 +100,7 @@ def has_missing_images(system_dir, extensions): json_output = json.dumps(list(systems_with_missing_images), indent=4) - output_file = os.path.join(settings["storagePath"], "/retrolibrary/cache/missing_systems.json") + output_file = os.path.join(storage_path, "/retrolibrary/cache/missing_systems.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) From c91a083c3e15533b37e6f7763aaa754431dde0ee Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 18:04:05 +0100 Subject: [PATCH 122/230] lovely / --- tools/retro-library/download_art.py | 2 +- tools/retro-library/download_art_platforms.py | 2 +- tools/retro-library/missing_artwork.py | 2 +- tools/retro-library/missing_artwork_platforms.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index c074b3ee4..82eb6b414 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -38,7 +38,7 @@ def getSettings(): # Path for the JSON and target folder from command-line arguments save_folder = sys.argv[1] -json_path = os.path.join(storage_path, "/retrolibrary/cache/missing_artwork.json") +json_path = os.path.join(storage_path, "retrolibrary/cache/missing_artwork.json") # Path for the log file home_dir = os.environ.get("HOME") diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index e43ae90df..11f4f5ed8 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -49,7 +49,7 @@ def log_message(message): def download_and_extract(output_dir): # Fixed path to the JSON file - json_file_path = os.path.join(storage_path, "/retrolibrary/cache/missing_systems.json") + json_file_path = os.path.join(storage_path, "retrolibrary/cache/missing_systems.json") # Check if the JSON file exists if not os.path.exists(json_file_path): diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 28a986598..42fb8e870 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -138,7 +138,7 @@ def collect_game_data(system_dir, extensions): # Save the JSON output json_output = json.dumps(game_list, indent=4) home_directory = os.path.expanduser("~") - output_file = os.path.join(storage_path, "/retrolibrary/cache/missing_artwork.json") + output_file = os.path.join(storage_path, "retrolibrary/cache/missing_artwork.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 0cfb0a0d6..7df0e7782 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -100,7 +100,7 @@ def has_missing_images(system_dir, extensions): json_output = json.dumps(list(systems_with_missing_images), indent=4) - output_file = os.path.join(storage_path, "/retrolibrary/cache/missing_systems.json") + output_file = os.path.join(storage_path, "retrolibrary/cache/missing_systems.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) From c458469f30bcbf96eb3a13d6b111b81d43c84cf8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 18:04:05 +0100 Subject: [PATCH 123/230] lovely / --- tools/retro-library/generate_game_lists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 4b25d1d8a..f0b440114 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -142,7 +142,7 @@ def collect_game_data(system_dir, extensions): json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) - output_file = os.path.join(storage_path, "/retrolibrary/cache/roms_games.json") + output_file = os.path.join(storage_path, "retrolibrary/cache/roms_games.json") os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: From 1729f70db5d617bad0284a23a778a6a2cdc7ca8a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 18:06:08 +0100 Subject: [PATCH 124/230] symlink --- functions/generateGameLists.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index b636ce2ff..d7c8beaba 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -9,6 +9,7 @@ generateGameLists() { mkdir -p "$storagePath/retrolibrary/artwork" mkdir -p "$accountfolder/config/grid/retrolibrary/" ln -s "$storagePath/retrolibrary/artwork/" "$accountfolder/config/grid/retrolibrary/artwork" + ln -s "$storagePath/retrolibrary/cache/" "$accountfolder/config/grid/retrolibrary/cache" generateGameLists_downloadAchievements generateGameLists_downloadData From a7c72b1fa6ff249018fbcbfa12116e8099669838 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 18:23:04 +0100 Subject: [PATCH 125/230] re --- tools/retro-library/download_art.py | 1 + tools/retro-library/download_art_platforms.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/retro-library/download_art.py b/tools/retro-library/download_art.py index 82eb6b414..03d49fa81 100644 --- a/tools/retro-library/download_art.py +++ b/tools/retro-library/download_art.py @@ -1,6 +1,7 @@ import json import requests import os +import re import sys from concurrent.futures import ThreadPoolExecutor, as_completed import subprocess diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 11f4f5ed8..059c24f03 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -5,6 +5,7 @@ import zipfile from io import BytesIO import sys +import re # Define the log file path home_dir = os.environ.get("HOME") From a08d3ac5f326a1485aee86b9d2b4b8fe8cf00dbc Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 18:24:59 +0100 Subject: [PATCH 126/230] skipped --- tools/retro-library/download_art_platforms.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 059c24f03..6e8115f32 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -82,9 +82,11 @@ def download_and_extract(output_dir): for platform in data: extracted_folder = os.path.join(output_dir, platform) if os.path.exists(extracted_folder): - log_message(f"Skipped: {platform} already extracted at {extracted_folder}.") - print(f"Skipped: {platform} already extracted at {extracted_folder}.") - continue + num_files = len([f for f in os.listdir(extracted_folder) if os.path.isfile(os.path.join(extracted_folder, f))]) + if num_files >= 3: + log_message(f"Skipped: {platform} already extracted at {extracted_folder} with {num_files} files.") + print(f"Skipped: {platform} already extracted at {extracted_folder} with {num_files} files.") + continue url = f"https://bot.emudeck.com/artwork_deck/{platform}.zip" log_message(f"Downloading: {platform}") From 60cf038d48239568fe50639ac295954e9ffce9e3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 25 Nov 2024 20:56:56 +0100 Subject: [PATCH 127/230] save states --- tools/retro-library/generate_game_lists.py | 46 +++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index f0b440114..3f41dc1d0 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -39,13 +39,53 @@ def getSettings(): settings = getSettings() storage_path = os.path.expandvars(settings["storagePath"]) - +saves_path = os.path.expandvars(settings["savesPath"]) # Function to write messages to the log file def log_message(message): with open(msg_file, "a") as log_file: # "a" to append messages without overwriting log_file.write(message + "\n") +def generate_saves_list(saves_path): + def clean_name(filename): + """Clean the game name using the same logic as in the ROM JSON.""" + name_cleaned = re.sub(r'\(.*?\)', '', filename) + name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) + name_cleaned = name_cleaned.strip().replace(' ', '_').replace('-', '_') + name_cleaned = re.sub(r'_+', '_', name_cleaned) + name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') + return name_cleaned + + saves_list = [] + states_dir = os.path.join(saves_path, "retroarch", "states") + + if not os.path.exists(states_dir): + log_message(f"Saves path does not exist: {states_dir}") + return + + for root, _, files in os.walk(states_dir): + for file in files: + file_path = os.path.join(root, file) + if os.path.isfile(file_path): + # Clean and prepare data for the JSON + cleaned_name = clean_name(os.path.splitext(file)[0]) + save_info = { + "name": cleaned_name, + "path": file_path + } + saves_list.append(save_info) + log_message(f"Found save state: {cleaned_name} at {file_path}") + + # Sort and write the JSON + saves_list_sorted = sorted(saves_list, key=lambda x: x['name']) + json_output = json.dumps(saves_list_sorted, indent=4) + + output_file = os.path.join(storage_path, "retrolibrary/cache/saves_states.json") + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'w') as f: + f.write(json_output) + log_message(f"Saved states JSON written to {output_file}") + def generate_game_lists(roms_path): def calculate_hash(file_path): """Calculate the MD5 hash of a file.""" @@ -154,3 +194,7 @@ def collect_game_data(system_dir, extensions): log_message("GGL: Starting game list generation...") generate_game_lists(f"{roms_path}") log_message("GGL: Game list generation completed.") + +log_message("GGL: Starting saves list generation...") +generate_saves_list(saves_path) +log_message("GGL: Saves list generation completed.") \ No newline at end of file From 5e719f46e88b6285fc5dee57c05f29d08cfc8297 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 26 Nov 2024 08:19:32 +0100 Subject: [PATCH 128/230] symlink data --- functions/generateGameLists.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index d7c8beaba..fc16aa462 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -147,6 +147,8 @@ generateGameLists_downloadAchievements(){ generateGameLists_downloadData(){ local folder="$storagePath/retrolibrary/data" + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) + ln -s "$folder" "$accountfolder/config/grid/retrolibrary/data" if [ ! -d $folder ]; then echo "Downloading Metada" > "$MSG" mkdir -p $folder @@ -154,8 +156,4 @@ generateGameLists_downloadData(){ cd $folder && unzip -o data.zip && rm data.zip echo "Metada Downloaded" > "$MSG" fi -} - -generateGameLists_readMessage(){ - cat "$MSG" -} +} \ No newline at end of file From 83f7eb83754c38dade8685ab7154d7a61af44d0d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 28 Nov 2024 11:58:00 +0100 Subject: [PATCH 129/230] Enable screenshots for savestates --- functions/EmuScripts/emuDeckRetroArch.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/functions/EmuScripts/emuDeckRetroArch.sh b/functions/EmuScripts/emuDeckRetroArch.sh index c52bebe1d..2af87f659 100644 --- a/functions/EmuScripts/emuDeckRetroArch.sh +++ b/functions/EmuScripts/emuDeckRetroArch.sh @@ -249,6 +249,11 @@ RetroArch_setupConfigurations(){ microphone_driverSetting="${microphone_driver}"\""sdl2"\" changeLine "$microphone_driver" "$microphone_driverSetting" "$RetroArch_configFile" + #Enable screenshots for savestates + savestate_thumbnail='savestate_thumbnail_enable = ' + savestate_thumbnailSetting="${input_driver}"\""true"\" + changeLine "$savestate_thumbnail" "$savestate_thumbnailSetting" "$RetroArch_configFile" + } RetroArch_buildbotDownloader(){ From 93a0f828a91e0b77485d8d4454a443947c6dd20d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 4 Dec 2024 19:46:07 +0100 Subject: [PATCH 130/230] fix reading key --- functions/ToolScripts/emuDeckCloudSync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/ToolScripts/emuDeckCloudSync.sh b/functions/ToolScripts/emuDeckCloudSync.sh index 031585c7f..8f1901f09 100644 --- a/functions/ToolScripts/emuDeckCloudSync.sh +++ b/functions/ToolScripts/emuDeckCloudSync.sh @@ -148,9 +148,9 @@ cloud_sync_setup_providers(){ json='{"token":"'"$token"'"}' - read cloud_key_id cloud_key < <(curl --request POST --url "https://token.emudeck.com/b2.php" \ + read -r cloud_key_id cloud_key < <(curl --request POST --url "https://token.emudeck.com/b2.php" \ --header "Content-Type: application/json" \ - -d "${json}" | jq -r '.cloud_key_id, .cloud_key') + -d "${json}" | jq -r '[.cloud_key_id, .cloud_key] | @tsv') "$cloud_sync_bin" config update "$cloud_sync_provider" key="$cloud_key" account="$cloud_key_id" From 1534e006d0d834213c8bb598833422cf02c2d8e4 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:06:31 +0100 Subject: [PATCH 131/230] Fix Cemu - No DSU --- .../controllerProfiles/Deck-Gamepad-Gyro.xml | 108 ++++--- .../data/cemu/controllerProfiles/Deck.xml | 106 +++---- .../cemu/controllerProfiles/controller0.xml | 267 ++++++++---------- .../data/cemu/controllerProfiles/deck2.xml | 121 -------- .../data/cemu/controllerProfiles/deck3.xml | 121 -------- .../data/cemu/controllerProfiles/deck4.xml | 121 -------- 6 files changed, 226 insertions(+), 618 deletions(-) delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml index 032fab6e6..778ffc21a 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml @@ -22,122 +22,118 @@ - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 11 - + 14 + - 12 + 1 - 13 - + 2 + - 14 + 3 - 15 - - - - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml index 64367f68a..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml @@ -1,120 +1,120 @@ - Wii U Pro Controller + Wii U GamePad Deck - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 12 - + 14 + - 13 + 1 - 14 - + 2 + - 15 + 3 - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml index bd22385db..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml @@ -1,146 +1,121 @@ - - - Wii U GamePad - Deck-Gamepad-Gyro - - DSUController - 0 - Controller 1 - true - - 0.25 - 1 - - - 0.25 - 1 - - - 0.25 - 1 - - 127.0.0.1 - 26760 - - - - XInput - 0 - Controller 1 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 11 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - + + + Wii U GamePad + Deck + + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller + 0 + + 0.25 + 1 + + + 0.25 + 1 + + + 0.25 + 1 + + + + 24 + + + + 23 + + + + 22 + + + + 21 + + + + 20 + + + + 19 + + + + 18 + + + + 17 + + + + 16 + + + + 15 + + + + 14 + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + 7 + + + + 8 + + + + 9 + + + + 10 + + + + 11 + + + + 12 + + + + 13 + + + + + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml deleted file mode 100644 index 13afb8a9b..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck2 - - XInput - 1 - Controller 2 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml deleted file mode 100644 index cfcc08ea0..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck3 - - XInput - 2 - Controller 3 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml deleted file mode 100644 index be5cbb21f..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck4 - - XInput - 3 - Controller 4 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - From e6e33eb815c4da2ed71232bc99257d4856efa495 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:06:31 +0100 Subject: [PATCH 132/230] Fix Cemu - No DSU --- .../controllerProfiles/Deck-Gamepad-Gyro.xml | 108 ++++--- .../data/cemu/controllerProfiles/Deck.xml | 106 +++---- .../cemu/controllerProfiles/controller0.xml | 267 ++++++++---------- .../data/cemu/controllerProfiles/deck2.xml | 121 -------- .../data/cemu/controllerProfiles/deck3.xml | 121 -------- .../data/cemu/controllerProfiles/deck4.xml | 121 -------- 6 files changed, 226 insertions(+), 618 deletions(-) delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml index 032fab6e6..778ffc21a 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml @@ -22,122 +22,118 @@ - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 11 - + 14 + - 12 + 1 - 13 - + 2 + - 14 + 3 - 15 - - - - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml index 64367f68a..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml @@ -1,120 +1,120 @@ - Wii U Pro Controller + Wii U GamePad Deck - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 12 - + 14 + - 13 + 1 - 14 - + 2 + - 15 + 3 - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml index bd22385db..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml @@ -1,146 +1,121 @@ - - - Wii U GamePad - Deck-Gamepad-Gyro - - DSUController - 0 - Controller 1 - true - - 0.25 - 1 - - - 0.25 - 1 - - - 0.25 - 1 - - 127.0.0.1 - 26760 - - - - XInput - 0 - Controller 1 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 11 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - + + + Wii U GamePad + Deck + + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller + 0 + + 0.25 + 1 + + + 0.25 + 1 + + + 0.25 + 1 + + + + 24 + + + + 23 + + + + 22 + + + + 21 + + + + 20 + + + + 19 + + + + 18 + + + + 17 + + + + 16 + + + + 15 + + + + 14 + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + 7 + + + + 8 + + + + 9 + + + + 10 + + + + 11 + + + + 12 + + + + 13 + + + + + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml deleted file mode 100644 index 13afb8a9b..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck2 - - XInput - 1 - Controller 2 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml deleted file mode 100644 index cfcc08ea0..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck3 - - XInput - 2 - Controller 3 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml deleted file mode 100644 index be5cbb21f..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck4 - - XInput - 3 - Controller 4 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - From 9d2273259d2be07bdc4be30cccb02f2cbfb866ab Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:06:31 +0100 Subject: [PATCH 133/230] Fix Cemu - No DSU --- .../controllerProfiles/Deck-Gamepad-Gyro.xml | 108 ++++--- .../data/cemu/controllerProfiles/Deck.xml | 106 +++---- .../cemu/controllerProfiles/controller0.xml | 267 ++++++++---------- .../data/cemu/controllerProfiles/deck2.xml | 121 -------- .../data/cemu/controllerProfiles/deck3.xml | 121 -------- .../data/cemu/controllerProfiles/deck4.xml | 121 -------- 6 files changed, 226 insertions(+), 618 deletions(-) delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml index 032fab6e6..778ffc21a 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml @@ -22,122 +22,118 @@ - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 11 - + 14 + - 12 + 1 - 13 - + 2 + - 14 + 3 - 15 - - - - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml index 64367f68a..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml @@ -1,120 +1,120 @@ - Wii U Pro Controller + Wii U GamePad Deck - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 12 - + 14 + - 13 + 1 - 14 - + 2 + - 15 + 3 - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml index bd22385db..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml @@ -1,146 +1,121 @@ - - - Wii U GamePad - Deck-Gamepad-Gyro - - DSUController - 0 - Controller 1 - true - - 0.25 - 1 - - - 0.25 - 1 - - - 0.25 - 1 - - 127.0.0.1 - 26760 - - - - XInput - 0 - Controller 1 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 11 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - + + + Wii U GamePad + Deck + + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller + 0 + + 0.25 + 1 + + + 0.25 + 1 + + + 0.25 + 1 + + + + 24 + + + + 23 + + + + 22 + + + + 21 + + + + 20 + + + + 19 + + + + 18 + + + + 17 + + + + 16 + + + + 15 + + + + 14 + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + 7 + + + + 8 + + + + 9 + + + + 10 + + + + 11 + + + + 12 + + + + 13 + + + + + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml deleted file mode 100644 index 13afb8a9b..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck2 - - XInput - 1 - Controller 2 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml deleted file mode 100644 index cfcc08ea0..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck3 - - XInput - 2 - Controller 3 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml deleted file mode 100644 index be5cbb21f..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck4 - - XInput - 3 - Controller 4 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - From c123a09f0dd314d7252a0986f3ad9d805a127cfe Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:06:31 +0100 Subject: [PATCH 134/230] Fix Cemu - No DSU --- .../controllerProfiles/Deck-Gamepad-Gyro.xml | 108 ++++--- .../data/cemu/controllerProfiles/Deck.xml | 106 +++---- .../cemu/controllerProfiles/controller0.xml | 267 ++++++++---------- .../data/cemu/controllerProfiles/deck2.xml | 121 -------- .../data/cemu/controllerProfiles/deck3.xml | 121 -------- .../data/cemu/controllerProfiles/deck4.xml | 121 -------- 6 files changed, 226 insertions(+), 618 deletions(-) delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml delete mode 100644 configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml index 032fab6e6..778ffc21a 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck-Gamepad-Gyro.xml @@ -22,122 +22,118 @@ - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 11 - + 14 + - 12 + 1 - 13 - + 2 + - 14 + 3 - 15 - - - - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml index 64367f68a..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/Deck.xml @@ -1,120 +1,120 @@ - Wii U Pro Controller + Wii U GamePad Deck - XInput - 0 - Controller 1 + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller 0 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 0.15 + 0.25 1 - 1 - + 24 + - 2 - + 23 + - 3 - + 22 + - 4 - + 21 + - 5 - + 20 + - 6 - + 19 + - 7 - + 18 + - 8 - + 17 + - 9 - + 16 + - 10 - + 15 + - 12 - + 14 + - 13 + 1 - 14 - + 2 + - 15 + 3 - 16 - + 4 + - 17 - + 5 + - 18 - + 6 + - 19 - + 7 + - 20 - + 8 + - 21 - + 9 + - 22 - + 10 + - 23 - + 11 + - 24 - + 12 + - 25 - + 13 + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml index bd22385db..1b0992ab3 100644 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml +++ b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/controller0.xml @@ -1,146 +1,121 @@ - - - Wii U GamePad - Deck-Gamepad-Gyro - - DSUController - 0 - Controller 1 - true - - 0.25 - 1 - - - 0.25 - 1 - - - 0.25 - 1 - - 127.0.0.1 - 26760 - - - - XInput - 0 - Controller 1 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 11 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - + + + Wii U GamePad + Deck + + SDLController + 0_030079f6de280000ff11000001000000 + Steam Deck Controller + 0 + + 0.25 + 1 + + + 0.25 + 1 + + + 0.25 + 1 + + + + 24 + + + + 23 + + + + 22 + + + + 21 + + + + 20 + + + + 19 + + + + 18 + + + + 17 + + + + 16 + + + + 15 + + + + 14 + + + + 1 + + + + 2 + + + + 3 + + + + 4 + + + + 5 + + + + 6 + + + + 7 + + + + 8 + + + + 9 + + + + 10 + + + + 11 + + + + 12 + + + + 13 + + + + + diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml deleted file mode 100644 index 13afb8a9b..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck2.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck2 - - XInput - 1 - Controller 2 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml deleted file mode 100644 index cfcc08ea0..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck3.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck3 - - XInput - 2 - Controller 3 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - diff --git a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml b/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml deleted file mode 100644 index be5cbb21f..000000000 --- a/configs/info.cemu.Cemu/data/cemu/controllerProfiles/deck4.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - Wii U Pro Controller - deck4 - - XInput - 3 - Controller 4 - 0 - - 0.15 - 1 - - - 0.15 - 1 - - - 0.15 - 1 - - - - 1 - - - - 2 - - - - 3 - - - - 4 - - - - 5 - - - - 6 - - - - 7 - - - - 8 - - - - 9 - - - - 10 - - - - 12 - - - - 13 - - - - 14 - - - - 15 - - - - 16 - - - - 17 - - - - 18 - - - - 19 - - - - 20 - - - - 21 - - - - 22 - - - - 23 - - - - 24 - - - - 25 - - - - - From be0a81ba20b02cc120707ccba4b499c35f60ee1a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:17:53 +0100 Subject: [PATCH 135/230] GreemDev --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index f61bc3719..2f166fcaf 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "ryujinx-mirror/ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" From a98fb5e9bbbba64ef9d8e6cc2142d3c7853017fb Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:23:24 +0100 Subject: [PATCH 136/230] path fix --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index 2f166fcaf..4cbfcafe9 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -51,7 +51,7 @@ Ryujinx_install(){ local showProgress=$1 if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" - tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" + tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" else return 1 From 3e7bdf6c3cd9ef39bf177d880ce1ee3f5c324d57 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:17:53 +0100 Subject: [PATCH 137/230] GreemDev --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index f61bc3719..2f166fcaf 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "ryujinx-mirror/ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" From 77c80a12b3e126035d26dae419b7b6ec75b0d127 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:23:24 +0100 Subject: [PATCH 138/230] path fix --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index 2f166fcaf..4cbfcafe9 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -51,7 +51,7 @@ Ryujinx_install(){ local showProgress=$1 if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" - tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" + tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" else return 1 From beddaf964a88c5ee0402191776895b21178e6641 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:17:53 +0100 Subject: [PATCH 139/230] GreemDev --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index f61bc3719..2f166fcaf 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "ryujinx-mirror/ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" From a12cecffc8bb9d3b41c63991148a5b18d1f5eec1 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:23:24 +0100 Subject: [PATCH 140/230] path fix --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index 2f166fcaf..4cbfcafe9 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -51,7 +51,7 @@ Ryujinx_install(){ local showProgress=$1 if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" - tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" + tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" else return 1 From 3b12ceb72251770b5b7b8b922dc2edffd942ef5b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:17:53 +0100 Subject: [PATCH 141/230] GreemDev --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index f61bc3719..2f166fcaf 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "ryujinx-mirror/ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" From 6260f87b8b1bdfb67f2bf091cd8e2e979254ae88 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Fri, 6 Dec 2024 11:23:24 +0100 Subject: [PATCH 142/230] path fix --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index 2f166fcaf..4cbfcafe9 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -51,7 +51,7 @@ Ryujinx_install(){ local showProgress=$1 if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" - tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications/publish" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" + tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" else return 1 From 710d0846ce94a8dad135cb471924f06b8eb54119 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 8 Dec 2024 11:29:46 +0100 Subject: [PATCH 143/230] temp 7z --- functions/ToolScripts/emuDeckPlugins.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/ToolScripts/emuDeckPlugins.sh b/functions/ToolScripts/emuDeckPlugins.sh index f2e54714d..fbc6b554f 100644 --- a/functions/ToolScripts/emuDeckPlugins.sh +++ b/functions/ToolScripts/emuDeckPlugins.sh @@ -136,21 +136,21 @@ Plugins_installDeckyRomLibrary(){ echo "Installing Decky Rom Library" local password=$1 local destinationFolder="$HOME/homebrew/plugins/decky-rom-library" - local DeckyControls_releaseURL="$(getLatestReleaseURLGH "EmuDeck/decky-rom-library" ".zip")" + local DeckyControls_releaseURL="$(getLatestReleaseURLGH "EmuDeck/decky-rom-library" ".7z")" mkdir -p "$HOME/homebrew/plugins/" if [ -d "$HOME/homebrew" ]; then Plugins_checkPassword $password echo $password | sudo -S rm -rf $destinationFolder - echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.zip" - echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.zip" -d "$HOME/homebrew/plugins/" && echo $password | sudo -S rm "$HOME/homebrew/plugins/decky-rom-library.zip" + echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.7z" + echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.7z" -d "$HOME/homebrew/plugins/" && echo $password | sudo -S rm "$HOME/homebrew/plugins/decky-rom-library.7z" echo $password | sudo -S chown $USER:$USER -R $HOME/homebrew/plugins/decky-rom-library echo $password | sudo -S chmod 555 -R $HOME/homebrew/plugins/decky-rom-library Plugins_install_cleanup $password else Plugins_installPluginLoader $password rm -rf $destinationFolder - echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.zip" - echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.zip" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/decky-rom-library.zip" + echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.7z" + echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.7z" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/decky-rom-library.7z" echo $password | sudo -S chown $USER:$USER -R $HOME/homebrew/plugins/decky-rom-library fi #RAachievemets From c701f45af11e5a2ed0fcdfc8f696ff080d444767 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 8 Dec 2024 11:30:33 +0100 Subject: [PATCH 144/230] emudeck cloud early --- functions/ToolScripts/emuDeckCloudSync.sh | 126 +++++++++++----------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/functions/ToolScripts/emuDeckCloudSync.sh b/functions/ToolScripts/emuDeckCloudSync.sh index 58598d702..ad31dd120 100644 --- a/functions/ToolScripts/emuDeckCloudSync.sh +++ b/functions/ToolScripts/emuDeckCloudSync.sh @@ -76,13 +76,13 @@ cloud_sync_config(){ #Check installation if [ ! -f "$cloud_sync_bin" ]; then - echo "false" + echo "false" elif [ ! -f "$cloud_sync_config" ]; then - echo "false" + echo "false" elif [ $cloud_sync_provider = '' ]; then - echo "false" + echo "false" else - echo "true" + echo "true" fi } @@ -144,15 +144,15 @@ cloud_sync_setup_providers(){ token="${token//---/|||}" user=$(echo $token | cut -d "|" -f 1) - setSetting cs_user "cs_$user/" + setSetting cs_user "cs$user/" json='{"token":"'"$token"'"}' - password=$(curl --request POST --url "https://token.emudeck.com/create-cs.php" --header "Content-Type: application/json" -d "${json}" | jq -r .cloud_token) - host="cloud.emudeck.com" - port="22" + read -r cloud_key_id cloud_key < <(curl --request POST --url "https://token.emudeck.com/b2.php" \ + --header "Content-Type: application/json" \ + -d "${json}" | jq -r '[.cloud_key_id, .cloud_key] | @tsv') - "$cloud_sync_bin" config update "$cloud_sync_provider" host="$host" user="cs_$user" port="$port" pass="$password" + "$cloud_sync_bin" config update "$cloud_sync_provider" key="$cloud_key" account="$cloud_key_id" "$cloud_sync_bin" mkdir "$cloud_sync_provider:"$cs_user"Emudeck/saves" cloud_sync_save_hash $savesPath @@ -254,34 +254,34 @@ cloud_sync_setup_providers(){ } cloud_sync_install_and_config(){ - #startLog ${FUNCNAME[0]} - local cloud_sync_provider=$1 + #startLog ${FUNCNAME[0]} + local cloud_sync_provider=$1 local token=$2 - #We force Chrome to be used as the default - { + #We force Chrome to be used as the default + { if [ $system != "darwin" ];then browser=$(xdg-settings get default-web-browser) - if [ "$browser" != 'com.google.Chrome.desktop' ];then - flatpak install flathub com.google.Chrome -y --user - xdg-settings set default-web-browser com.google.Chrome.desktop - fi + if [ "$browser" != 'com.google.Chrome.desktop' ];then + flatpak install flathub com.google.Chrome -y --user + xdg-settings set default-web-browser com.google.Chrome.desktop + fi - if [ ! -f "$cloud_sync_bin" ]; then - cloud_sync_install $cloud_sync_provider - fi + if [ ! -f "$cloud_sync_bin" ]; then + cloud_sync_install $cloud_sync_provider + fi fi - } && cloud_sync_config "$cloud_sync_provider" "$token" + } && cloud_sync_config "$cloud_sync_provider" "$token" - setSetting cloud_sync_provider "$cloud_sync_provider" - setSetting cloud_sync_status "true" + setSetting cloud_sync_provider "$cloud_sync_provider" + setSetting cloud_sync_status "true" - #We get the previous default browser back + #We get the previous default browser back if [ $system != "darwin" ];then - if [ "$browser" != 'com.google.Chrome.desktop' ];then - xdg-settings set default-web-browser $browser - fi + if [ "$browser" != 'com.google.Chrome.desktop' ];then + xdg-settings set default-web-browser $browser + fi fi } @@ -332,12 +332,12 @@ cloud_sync_upload(){ } cloud_sync_download(){ - local branch=$(cd "$HOME"/.config/EmuDeck/backend && git rev-parse --abbrev-ref HEAD) - if [[ "$branch" == *"early"* ]] || [ "$branch" == "dev" ] ; then - echo "CloudSync Downloading" - else - return 0 - fi + local branch=$(cd "$HOME"/.config/EmuDeck/backend && git rev-parse --abbrev-ref HEAD) + if [[ "$branch" == *"early"* ]] || [ "$branch" == "dev" ] ; then + echo "CloudSync Downloading" + else + return 0 + fi # startLog ${FUNCNAME[0]} local emuName=$1 local timestamp=$(date +%s) @@ -443,22 +443,22 @@ cloud_sync_uploadEmu(){ #rm -rf $savesPath/$emuName/.pending_upload cloud_sync_createBackup "$emuName" cloud_sync_download $emuName && { - rm -rf $savesPath/$emuName/.fail_download > /dev/null - rm -rf $savesPath/$emuName/.pending_download > /dev/null - rm -rf $savesPath/$emuName/.fail_upload > /dev/null - rm -rf $savesPath/$emuName/.pending_upload > /dev/null - } + rm -rf $savesPath/$emuName/.fail_download > /dev/null + rm -rf $savesPath/$emuName/.pending_download > /dev/null + rm -rf $savesPath/$emuName/.fail_upload > /dev/null + rm -rf $savesPath/$emuName/.pending_upload > /dev/null + } elif [[ $response =~ "0-" ]]; then #Upload - OK #rm -rf $savesPath/$emuName/.pending_upload cloud_sync_createBackup "$emuName" cloud_sync_upload $emuName && { - rm -rf $savesPath/$emuName/.fail_download > /dev/null - rm -rf $savesPath/$emuName/.pending_download > /dev/null - rm -rf $savesPath/$emuName/.fail_upload > /dev/null - rm -rf $savesPath/$emuName/.pending_upload > /dev/null - } + rm -rf $savesPath/$emuName/.fail_download > /dev/null + rm -rf $savesPath/$emuName/.pending_download > /dev/null + rm -rf $savesPath/$emuName/.fail_upload > /dev/null + rm -rf $savesPath/$emuName/.pending_upload > /dev/null + } else #Skip - Cancel return @@ -525,11 +525,11 @@ cloud_sync_downloadEmu(){ #Download - OK cloud_sync_createBackup "$emuName" cloud_sync_download $emuName && { - rm -rf $savesPath/$emuName/.fail_download > /dev/null - rm -rf $savesPath/$emuName/.pending_download > /dev/null - rm -rf $savesPath/$emuName/.fail_upload > /dev/null - rm -rf $savesPath/$emuName/.pending_upload > /dev/null - } + rm -rf $savesPath/$emuName/.fail_download > /dev/null + rm -rf $savesPath/$emuName/.pending_download > /dev/null + rm -rf $savesPath/$emuName/.fail_upload > /dev/null + rm -rf $savesPath/$emuName/.pending_upload > /dev/null + } #echo $timestamp > "$savesPath"/$emuName/.pending_upload elif [[ $response =~ "0-" ]]; then @@ -537,11 +537,11 @@ cloud_sync_downloadEmu(){ #rm -rf $savesPath/$emuName/.pending_upload cloud_sync_createBackup "$emuName" cloud_sync_upload $emuName && { - rm -rf $savesPath/$emuName/.fail_download > /dev/null - rm -rf $savesPath/$emuName/.pending_download > /dev/null - rm -rf $savesPath/$emuName/.fail_upload > /dev/null - rm -rf $savesPath/$emuName/.pending_upload > /dev/null - } + rm -rf $savesPath/$emuName/.fail_download > /dev/null + rm -rf $savesPath/$emuName/.pending_download > /dev/null + rm -rf $savesPath/$emuName/.fail_upload > /dev/null + rm -rf $savesPath/$emuName/.pending_upload > /dev/null + } else #Skip - Cancel return @@ -572,20 +572,20 @@ cloud_sync_downloadEmu(){ #rm -rf $savesPath/$emuName/.pending_upload cloud_sync_createBackup "$emuName" cloud_sync_upload $emuName && { - rm -rf $savesPath/$emuName/.fail_download > /dev/null - rm -rf $savesPath/$emuName/.pending_download > /dev/null - rm -rf $savesPath/$emuName/.fail_upload > /dev/null - rm -rf $savesPath/$emuName/.pending_upload > /dev/null - } + rm -rf $savesPath/$emuName/.fail_download > /dev/null + rm -rf $savesPath/$emuName/.pending_download > /dev/null + rm -rf $savesPath/$emuName/.fail_upload > /dev/null + rm -rf $savesPath/$emuName/.pending_upload > /dev/null + } elif [[ $response =~ "0-" ]]; then #Download - OK cloud_sync_createBackup "$emuName" cloud_sync_download $emuName && { - rm -rf $savesPath/$emuName/.fail_download > /dev/null - rm -rf $savesPath/$emuName/.pending_download > /dev/null - rm -rf $savesPath/$emuName/.fail_upload > /dev/null - rm -rf $savesPath/$emuName/.pending_upload > /dev/null - } + rm -rf $savesPath/$emuName/.fail_download > /dev/null + rm -rf $savesPath/$emuName/.pending_download > /dev/null + rm -rf $savesPath/$emuName/.fail_upload > /dev/null + rm -rf $savesPath/$emuName/.pending_upload > /dev/null + } else #Skip - Cancel return From 95697d858b6214fc8940ed57a2adc8a0f771ee7a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 12 Dec 2024 10:37:56 +0100 Subject: [PATCH 145/230] fix cloudHealth --- functions/cloudSyncHealth.sh | 37 +++++++----------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/functions/cloudSyncHealth.sh b/functions/cloudSyncHealth.sh index 81ae1d1f6..19fdb7108 100644 --- a/functions/cloudSyncHealth.sh +++ b/functions/cloudSyncHealth.sh @@ -55,31 +55,9 @@ cloudSyncHealth(){ #Zenity asking SRM or ESDE #Opening RA/ESDE in background kill="ESDE" - zenity --question --title "CloudSync Health" --text "Do you launch your games using EmulationStation?" --cancel-label "No" --ok-label "Yes" + zenity --question --title "CloudSync Health" --text "You need to have RetroArch installed for this to work." --cancel-label "Cancel" --ok-label "OK" if [ $? = 0 ]; then - notify-send "ESDE" --icon="$HOME/.local/share/icons/emudeck/EmuDeck.png" --app-name "EmuDeck CloudSync" - touch "$savesPath/.gaming" - touch "$savesPath/.watching" - echo "all" > "$savesPath/.emuName" - cloud_sync_startService - - systemctl --user is-active --quiet "EmuDeckCloudSync.service" - status=$? - - if [ $status -eq 0 ]; then - echo "CloudSync Service running" - watcherStatus=0 - else - text="$(printf "CloudSync Error.\nCloudSync service is not running. Please reinstall CloudSync and try again")" - zenity --error \ - --title="EmuDeck" \ - --width=400 \ - --text="${text}" 2>/dev/null - fi - - "$ESDE_toolPath" & xdotool search --sync --onlyvisible --name '^ES-DE$' windowminimize - else kill="RETROARCH" notify-send "RETROARCH" --icon="$HOME/.local/share/icons/emudeck/EmuDeck.png" --app-name "EmuDeck CloudSync" touch "$savesPath/.gaming" @@ -102,6 +80,10 @@ cloudSyncHealth(){ fi /usr/bin/flatpak run org.libretro.RetroArch & xdotool search --sync --name '^RetroArch$' windowminimize + else + zenity --info --width=400 --text="Please install RetroArch from Manage Emulators..." + + exit fi @@ -164,11 +146,8 @@ cloudSyncHealth(){ #Delete remote test file "$cloud_sync_bin" delete "$cloud_sync_provider":"$cs_user"Emudeck/saves/retroarch/test_emudeck.txt - if [ $kill == "RETROARCH" ];then - killall retroarch - else - xdotool search --sync --name '^ES-DE$' windowquit - fi + killall retroarch + } > "$HOME/emudeck/logs/cloudHealth.log" @@ -232,7 +211,6 @@ cloudSyncHealth(){ echo "Save folder not found" else echo "Upload Status: Failure" - echo "" fi echo "" @@ -244,7 +222,6 @@ cloudSyncHealth(){ echo "Save folder not found" else echo "Download Status: Failure" - echo "" fi echo "" From 7d52182afc14afd7318840246374f5b3eef520eb Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 12 Dec 2024 10:37:56 +0100 Subject: [PATCH 146/230] fix cloudHealth --- functions/cloudSyncHealth.sh | 37 +++++++----------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/functions/cloudSyncHealth.sh b/functions/cloudSyncHealth.sh index 81ae1d1f6..19fdb7108 100644 --- a/functions/cloudSyncHealth.sh +++ b/functions/cloudSyncHealth.sh @@ -55,31 +55,9 @@ cloudSyncHealth(){ #Zenity asking SRM or ESDE #Opening RA/ESDE in background kill="ESDE" - zenity --question --title "CloudSync Health" --text "Do you launch your games using EmulationStation?" --cancel-label "No" --ok-label "Yes" + zenity --question --title "CloudSync Health" --text "You need to have RetroArch installed for this to work." --cancel-label "Cancel" --ok-label "OK" if [ $? = 0 ]; then - notify-send "ESDE" --icon="$HOME/.local/share/icons/emudeck/EmuDeck.png" --app-name "EmuDeck CloudSync" - touch "$savesPath/.gaming" - touch "$savesPath/.watching" - echo "all" > "$savesPath/.emuName" - cloud_sync_startService - - systemctl --user is-active --quiet "EmuDeckCloudSync.service" - status=$? - - if [ $status -eq 0 ]; then - echo "CloudSync Service running" - watcherStatus=0 - else - text="$(printf "CloudSync Error.\nCloudSync service is not running. Please reinstall CloudSync and try again")" - zenity --error \ - --title="EmuDeck" \ - --width=400 \ - --text="${text}" 2>/dev/null - fi - - "$ESDE_toolPath" & xdotool search --sync --onlyvisible --name '^ES-DE$' windowminimize - else kill="RETROARCH" notify-send "RETROARCH" --icon="$HOME/.local/share/icons/emudeck/EmuDeck.png" --app-name "EmuDeck CloudSync" touch "$savesPath/.gaming" @@ -102,6 +80,10 @@ cloudSyncHealth(){ fi /usr/bin/flatpak run org.libretro.RetroArch & xdotool search --sync --name '^RetroArch$' windowminimize + else + zenity --info --width=400 --text="Please install RetroArch from Manage Emulators..." + + exit fi @@ -164,11 +146,8 @@ cloudSyncHealth(){ #Delete remote test file "$cloud_sync_bin" delete "$cloud_sync_provider":"$cs_user"Emudeck/saves/retroarch/test_emudeck.txt - if [ $kill == "RETROARCH" ];then - killall retroarch - else - xdotool search --sync --name '^ES-DE$' windowquit - fi + killall retroarch + } > "$HOME/emudeck/logs/cloudHealth.log" @@ -232,7 +211,6 @@ cloudSyncHealth(){ echo "Save folder not found" else echo "Upload Status: Failure" - echo "" fi echo "" @@ -244,7 +222,6 @@ cloudSyncHealth(){ echo "Save folder not found" else echo "Download Status: Failure" - echo "" fi echo "" From 81d63bc8e112a2dbceba79450a3d710e364347d8 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 14 Dec 2024 23:21:43 +0100 Subject: [PATCH 147/230] EmuDecky password for bazzite --- functions/ToolScripts/emuDeckPlugins.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/functions/ToolScripts/emuDeckPlugins.sh b/functions/ToolScripts/emuDeckPlugins.sh index fbc6b554f..027e1be99 100644 --- a/functions/ToolScripts/emuDeckPlugins.sh +++ b/functions/ToolScripts/emuDeckPlugins.sh @@ -7,14 +7,14 @@ Plugins_install_cleanup() { #systemctl restart plugin_loader #Deleting temp password - if [ "$password" = "Decky!" ]; then + if [ "$password" = "EmuDecky!" ]; then echo "$password" | sudo -S -k passwd -d $(whoami) && echo "true" fi } Plugins_checkPassword(){ local password=$1 - if [ "$password" = "Decky!" ]; then + if [ "$password" = "EmuDecky!" ]; then #We create the password yes "$password" | passwd $(whoami) &>/dev/null elif [ "$system" == "chimeraos" ]; then @@ -60,14 +60,13 @@ Plugins_installPowerTools(){ Plugins_checkPassword $password ptHash=$(curl https://beta.deckbrew.xyz/plugins | jq -r '.[] | select(.name=="PowerTools").versions[0].hash') local url="https://cdn.tzatzikiweeb.moe/file/steam-deck-homebrew/versions/$ptHash.zip" - + Plugins_installPluginLoader $password if [ -d "$HOME/homebrew" ]; then echo $password | sudo -S rm -rf "$HOME/homebrew/plugins/PowerTools" echo $password | sudo -S curl -l "$url" --output "$HOME/homebrew/PowerTools.zip.temp" && mv "$HOME/homebrew/PowerTools.zip.temp" "$HOME/homebrew/PowerTools.zip" echo $password | sudo -S unzip "$HOME/homebrew/PowerTools.zip" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/PowerTools.zip" Plugins_install_cleanup $password else - Plugins_installPluginLoader $password rm -rf "$HOME/homebrew/plugins/PowerTools" echo $password | sudo -S curl -l "$url" --output "$HOME/homebrew/PowerTools.zip.temp" && mv "$HOME/homebrew/PowerTools.zip.temp" "$HOME/homebrew/PowerTools.zip" echo $password | sudo -S unzip "$HOME/homebrew/PowerTools.zip" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/PowerTools.zip" @@ -80,6 +79,7 @@ Plugins_installPowerControl(){ local destinationFolder="$HOME/homebrew/plugins/PowerControl" local PowerControl_releaseURL="$(getLatestReleaseURLGH "mengmeet/PowerControl" ".tar.gz")" mkdir -p "$HOME/homebrew/plugins/" + Plugins_installPluginLoader $password if [ -d "$HOME/homebrew" ]; then Plugins_checkPassword $password echo $password | sudo -S rm -rf $destinationFolder @@ -89,7 +89,6 @@ Plugins_installPowerControl(){ echo $password | sudo -S chmod 555 -R $HOME/homebrew/plugins/PowerControl Plugins_install_cleanup $password else - Plugins_installPluginLoader $password rm -rf $destinationFolder echo $password | sudo -S curl -L "$PowerControl_releaseURL" -o "$HOME/homebrew/plugins/PowerControl.tar.gz" echo $password | sudo -S unzip "$HOME/homebrew/plugins/PowerControl.tar.gz" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/PowerControl.tar.gz" @@ -109,7 +108,7 @@ Plugins_installEmuDecky(){ local DeckyControls_releaseURL="$(getLatestReleaseURLGH "EmuDeck/EmuDecky" ".zip")" mkdir -p "$HOME/homebrew/plugins/" - + Plugins_installPluginLoader $password if [ -d "$HOME/homebrew" ]; then Plugins_checkPassword $password echo $password | sudo -S rm -rf $destinationFolder @@ -119,7 +118,6 @@ Plugins_installEmuDecky(){ echo $password | sudo -S chmod 555 -R $HOME/homebrew/plugins/EmuDecky Plugins_install_cleanup $password else - Plugins_installPluginLoader $password rm -rf $destinationFolder echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/EmuDecky.zip" echo $password | sudo -S unzip "$HOME/homebrew/plugins/EmuDecky.zip" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/EmuDecky.zip" @@ -138,6 +136,7 @@ Plugins_installDeckyRomLibrary(){ local destinationFolder="$HOME/homebrew/plugins/decky-rom-library" local DeckyControls_releaseURL="$(getLatestReleaseURLGH "EmuDeck/decky-rom-library" ".7z")" mkdir -p "$HOME/homebrew/plugins/" + Plugins_installPluginLoader $password if [ -d "$HOME/homebrew" ]; then Plugins_checkPassword $password echo $password | sudo -S rm -rf $destinationFolder @@ -147,7 +146,6 @@ Plugins_installDeckyRomLibrary(){ echo $password | sudo -S chmod 555 -R $HOME/homebrew/plugins/decky-rom-library Plugins_install_cleanup $password else - Plugins_installPluginLoader $password rm -rf $destinationFolder echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.7z" echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.7z" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/decky-rom-library.7z" From 91011a6352cfbdc5cb69a5973dd97698a7d80365 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 15 Dec 2024 01:19:27 +0100 Subject: [PATCH 148/230] lowercase names for pictures --- tools/retro-library/generate_game_lists.py | 2 +- tools/retro-library/missing_artwork.py | 2 +- tools/retro-library/missing_artwork_platforms.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 3f41dc1d0..fe33e5f63 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -127,7 +127,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') name_cleaned_pegasus = name.replace(',_', ',') - + name_cleaned = name_cleaned.lower() # Calculate the ROM hash rom_hash = calculate_hash(file_path) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 42fb8e870..770f84f30 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -86,7 +86,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = name_cleaned.strip().replace(' ', '_').replace('-', '_') name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - + name_cleaned = name_cleaned.lower() rom_hash = calculate_hash(file_path) # Check for missing images diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 7df0e7782..f3b6a8467 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -64,7 +64,7 @@ def has_missing_images(system_dir, extensions): name_cleaned = name_cleaned.replace(' ', '_').replace('-', '_') name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') - + name_cleaned = name_cleaned.lower() img_path = os.path.join(images_path, f"{platform}/{name_cleaned}.jpg") if not os.path.exists(img_path): return True From 3a63f5507a4d726c199e9a84de05f342f6700272 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 16 Dec 2024 05:16:06 +0100 Subject: [PATCH 149/230] Update emuDeckXenia.sh --- functions/EmuScripts/emuDeckXenia.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckXenia.sh b/functions/EmuScripts/emuDeckXenia.sh index 9752adb4f..54a2f5627 100644 --- a/functions/EmuScripts/emuDeckXenia.sh +++ b/functions/EmuScripts/emuDeckXenia.sh @@ -29,7 +29,7 @@ Xenia_install(){ #need to look at standardizing exe name; or download both? let the user choose at runtime? #curl -L "$Xenia_releaseURL" --output "$romsPath"/xbox360/xenia.zip - if safeDownload "$Xenia_emuName" "$Xenia_releaseURL" "$romsPath/xbox360/xenia.zip" "$showProgress"; then + if safeDownload "$Xenia_emuName" "https://github.com/xenia-canary/xenia-canary/releases/download/eb99874/xenia_canary.zip" "$romsPath/xbox360/xenia.zip" "$showProgress"; then #mkdir -p "$romsPath"/xbox360/tmp unzip -o "$romsPath"/xbox360/xenia.zip -d "$romsPath"/xbox360 #rsync -avzh "$romsPath"/xbox360/tmp/ "$romsPath"/xbox360/ From 365a6e7f7103fcb0eadcaee5bed0ce9a2827c77a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 16 Dec 2024 05:16:06 +0100 Subject: [PATCH 150/230] Update emuDeckXenia.sh --- functions/EmuScripts/emuDeckXenia.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckXenia.sh b/functions/EmuScripts/emuDeckXenia.sh index 9752adb4f..54a2f5627 100644 --- a/functions/EmuScripts/emuDeckXenia.sh +++ b/functions/EmuScripts/emuDeckXenia.sh @@ -29,7 +29,7 @@ Xenia_install(){ #need to look at standardizing exe name; or download both? let the user choose at runtime? #curl -L "$Xenia_releaseURL" --output "$romsPath"/xbox360/xenia.zip - if safeDownload "$Xenia_emuName" "$Xenia_releaseURL" "$romsPath/xbox360/xenia.zip" "$showProgress"; then + if safeDownload "$Xenia_emuName" "https://github.com/xenia-canary/xenia-canary/releases/download/eb99874/xenia_canary.zip" "$romsPath/xbox360/xenia.zip" "$showProgress"; then #mkdir -p "$romsPath"/xbox360/tmp unzip -o "$romsPath"/xbox360/xenia.zip -d "$romsPath"/xbox360 #rsync -avzh "$romsPath"/xbox360/tmp/ "$romsPath"/xbox360/ From 7d7e1925f37e77af73fd9123d33095904e0fa2d3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 16 Dec 2024 09:57:43 +0100 Subject: [PATCH 151/230] new expand --- tools/retro-library/download_art_platforms.py | 11 ++++++++--- tools/retro-library/generate_game_lists.py | 11 ++++++++--- tools/retro-library/missing_artwork.py | 11 ++++++++--- tools/retro-library/missing_artwork_platforms.py | 14 ++++++++------ 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/tools/retro-library/download_art_platforms.py b/tools/retro-library/download_art_platforms.py index 6e8115f32..aa98325e3 100644 --- a/tools/retro-library/download_art_platforms.py +++ b/tools/retro-library/download_art_platforms.py @@ -14,10 +14,12 @@ def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") + if os.name == 'nt': config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') else: config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} with open(config_file_path, 'r') as file: @@ -25,13 +27,16 @@ def getSettings(): match = pattern.search(line) if match: variable = match.group(1) - value = match.group(2).strip('"') - configuration[variable] = value + value = match.group(2).strip().strip('"') + expanded_value = os.path.expandvars(value.replace('"', '').replace("'", "")) + configuration[variable] = expanded_value + # Obtener rama actual del repositorio backend if os.name == 'nt': - bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + bash_command = f"cd {os.path.join(user_home, 'AppData', 'Roaming', 'EmuDeck', 'backend')} && git rev-parse --abbrev-ref HEAD" else: bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) configuration["branch"] = result.stdout.strip() diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index fe33e5f63..1bd8a59ed 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -12,10 +12,12 @@ def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") + if os.name == 'nt': config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') else: config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} with open(config_file_path, 'r') as file: @@ -23,13 +25,16 @@ def getSettings(): match = pattern.search(line) if match: variable = match.group(1) - value = match.group(2).strip('"') - configuration[variable] = value + value = match.group(2).strip().strip('"') + expanded_value = os.path.expandvars(value.replace('"', '').replace("'", "")) + configuration[variable] = expanded_value + # Obtener rama actual del repositorio backend if os.name == 'nt': - bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + bash_command = f"cd {os.path.join(user_home, 'AppData', 'Roaming', 'EmuDeck', 'backend')} && git rev-parse --abbrev-ref HEAD" else: bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) configuration["branch"] = result.stdout.strip() diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 770f84f30..6d2780a56 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -12,10 +12,12 @@ def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") + if os.name == 'nt': config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') else: config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} with open(config_file_path, 'r') as file: @@ -23,13 +25,16 @@ def getSettings(): match = pattern.search(line) if match: variable = match.group(1) - value = match.group(2).strip('"') - configuration[variable] = value + value = match.group(2).strip().strip('"') + expanded_value = os.path.expandvars(value.replace('"', '').replace("'", "")) + configuration[variable] = expanded_value + # Obtener rama actual del repositorio backend if os.name == 'nt': - bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + bash_command = f"cd {os.path.join(user_home, 'AppData', 'Roaming', 'EmuDeck', 'backend')} && git rev-parse --abbrev-ref HEAD" else: bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) configuration["branch"] = result.stdout.strip() diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index f3b6a8467..0aa5127b8 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -10,10 +10,12 @@ def getSettings(): pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') user_home = os.path.expanduser("~") + if os.name == 'nt': config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') else: config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + configuration = {} with open(config_file_path, 'r') as file: @@ -21,13 +23,16 @@ def getSettings(): match = pattern.search(line) if match: variable = match.group(1) - value = match.group(2).strip('"') - configuration[variable] = value + value = match.group(2).strip().strip('"') + expanded_value = os.path.expandvars(value.replace('"', '').replace("'", "")) + configuration[variable] = expanded_value + # Obtener rama actual del repositorio backend if os.name == 'nt': - bash_command = f"cd {appdata_roaming_path}/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + bash_command = f"cd {os.path.join(user_home, 'AppData', 'Roaming', 'EmuDeck', 'backend')} && git rev-parse --abbrev-ref HEAD" else: bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) configuration["branch"] = result.stdout.strip() @@ -35,9 +40,6 @@ def getSettings(): return configuration -settings = getSettings() -storage_path = os.path.expandvars(settings["storagePath"]) - # Función para escribir en el archivo de log def log_message(message): From 38a1dc414a37728420f75c11aa5d20b7496884f3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 16 Dec 2024 16:55:15 +0100 Subject: [PATCH 152/230] fix rclone cloud --- configs/rclone/rclone.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/configs/rclone/rclone.conf b/configs/rclone/rclone.conf index 8b7f90705..a480a29ce 100644 --- a/configs/rclone/rclone.conf +++ b/configs/rclone/rclone.conf @@ -46,8 +46,7 @@ user = pass = [Emudeck-cloud] -type = sftp -host = -user = -pass = -port = \ No newline at end of file +type = b2 +account = +key = +hard_delete = false \ No newline at end of file From 9437a2e24b30e4e2ee179031fe33d46b592262ac Mon Sep 17 00:00:00 2001 From: Glenn Hope Date: Tue, 17 Dec 2024 00:38:56 +0800 Subject: [PATCH 153/230] Fix install.sh to run prerequisite checks on all distros (#1347) * fixes dolphin control issues * Revert "fixes dolphin control issues" This reverts commit 32945b947a0306418d563386cd6c0e17030cf6c4. * dolphin for real * versions * forced citra * Fix boolean logic in install.sh * Switch to only "not steamos" --------- Co-authored-by: Dragoon Dorise --- install.sh | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/install.sh b/install.sh index 34f511ae5..eb5509980 100644 --- a/install.sh +++ b/install.sh @@ -2,25 +2,15 @@ linuxID=$(lsb_release -si) sandbox="" -if [ linuxID = "Ubuntu" ]; then + +if [ $linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi -#Python PIP for steamOS -#if [ $linuxID == "SteamOS" ]; then -# python -m pip --version &> /dev/null || python -m ensurepip --upgrade -# python -m pip install --upgrade pip -# grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc -# grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc -# PATH="$HOME/.local/bin:$PATH" -#fi - -if [ $linuxID != "ChimeraOS" ]; then - -echo "installing EmuDeck" - -elif [ $linuxID != "SteamOS" ]; then +if [ "$linuxID" == "SteamOS" ]; then + echo "installing EmuDeck" +else zenityAvailable=$(command -v zenity &> /dev/null && echo true) if [[ $zenityAvailable = true ]];then @@ -121,4 +111,4 @@ mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage ~/Applications/EmuDeck.AppImage $sandbox -exit \ No newline at end of file +exit From 9dab4cb91f1c05a3c4a9874eecba0c2149baddbd Mon Sep 17 00:00:00 2001 From: Gabriel Cruz <48037228+gabrielpcruz@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:44:43 -0300 Subject: [PATCH 154/230] Update install.sh (#1369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixes dolphin control issues * Revert "fixes dolphin control issues" This reverts commit 32945b947a0306418d563386cd6c0e17030cf6c4. * dolphin for real * versions * forced citra * Fix Cemu - No DSU * GreemDev * path fix * Update install.sh On Ubuntu 24.04.1 LTS is getting this error: installing EmuDeck [36436:1215/124135.236209:FATAL:setuid_sandbox_host.cc(157)] The SUID sandbox helper binary was found, but is not configured correctly. Rather than run without sandboxing I'm aborting now. You need to make sure that /tmp/.mount_EmuDecSYLXPs/chrome-sandbox is owned by root and has mode 4755. bash, linha 111: 36436 Trace/breakpoint trap (imagem do núcleo gravada) ~/Applications/EmuDeck.AppImage $sandbox Something went wrong! Error at 111 NULL: ~/Applications/EmuDeck.AppImage $sandbox I just add an verification for set sandbox to "--no-sandbox" and it works! * Update install.sh Update --------- Co-authored-by: Dragoon Dorise --- install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/install.sh b/install.sh index eb5509980..d5d3b6635 100644 --- a/install.sh +++ b/install.sh @@ -7,7 +7,6 @@ if [ $linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi - if [ "$linuxID" == "SteamOS" ]; then echo "installing EmuDeck" else From 23770ebb33f78a515c83b29a5ed953286552b336 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 16 Dec 2024 17:46:25 +0100 Subject: [PATCH 155/230] fix prerequisites installation --- install-early.sh | 23 +++++++---------------- install-unstable.sh | 23 +++++++---------------- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/install-early.sh b/install-early.sh index 6da5667d0..a5632e0f0 100644 --- a/install-early.sh +++ b/install-early.sh @@ -2,24 +2,14 @@ linuxID=$(lsb_release -si) sandbox="" -if [ linuxID = "Ubuntu" ]; then + +if [ $linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi -#Python PIP for steamOS -#if [ $linuxID == "SteamOS" ]; then -# python -m pip --version &> /dev/null || python -m ensurepip --upgrade -# python -m pip install --upgrade pip -# grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc -# grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc -# PATH="$HOME/.local/bin:$PATH" -#fi -if [ $linuxID != "ChimeraOS" ]; then - -echo "installing EmuDeck" - -elif [ $linuxID != "SteamOS" ]; then - +if [ "$linuxID" == "SteamOS" ]; then + echo "installing EmuDeck" +else zenityAvailable=$(command -v zenity &> /dev/null && echo true) if [[ $zenityAvailable = true ]];then @@ -63,6 +53,7 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" + sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS @@ -119,4 +110,4 @@ mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage ~/Applications/EmuDeck.AppImage $sandbox -exit \ No newline at end of file +exit diff --git a/install-unstable.sh b/install-unstable.sh index 612aec489..21ee999ab 100644 --- a/install-unstable.sh +++ b/install-unstable.sh @@ -2,24 +2,14 @@ linuxID=$(lsb_release -si) sandbox="" -if [ linuxID = "Ubuntu" ]; then + +if [ $linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi -#Python PIP for steamOS -#if [ $linuxID == "SteamOS" ]; then -# python -m pip --version &> /dev/null || python -m ensurepip --upgrade -# python -m pip install --upgrade pip -# grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc -# grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc -# PATH="$HOME/.local/bin:$PATH" -#fi -if [ $linuxID != "ChimeraOS" ]; then - -echo "installing EmuDeck" - -elif [ $linuxID != "SteamOS" ]; then - +if [ "$linuxID" == "SteamOS" ]; then + echo "installing EmuDeck" +else zenityAvailable=$(command -v zenity &> /dev/null && echo true) if [[ $zenityAvailable = true ]];then @@ -63,6 +53,7 @@ elif [ $linuxID != "SteamOS" ]; then if command -v apt-get >/dev/null; then echo "Installing packages with apt..." DEBIAN_DEPS="jq zenity flatpak unzip bash libfuse2 git rsync whiptail python" + sudo killall apt apt-get sudo apt-get -y update sudo apt-get -y install $DEBIAN_DEPS @@ -119,4 +110,4 @@ mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage ~/Applications/EmuDeck.AppImage $sandbox -exit \ No newline at end of file +exit From 239182b13971decb759973ee5c2b4bd290ea1644 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 17 Dec 2024 20:55:16 +0100 Subject: [PATCH 156/230] ryujinx-mirror is back --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index 4cbfcafe9..e52e32426 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "ryujinx-mirror/ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" From fb6c0d541b261aabb88317c628a47ff226a3d49b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Tue, 17 Dec 2024 20:59:10 +0100 Subject: [PATCH 157/230] Update emuDeckRyujinx.sh --- functions/EmuScripts/emuDeckRyujinx.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index 4cbfcafe9..f9b3a553c 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "ryujinx-mirror/ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" @@ -312,4 +312,4 @@ Ryujinx_flushEmulatorLauncher(){ flushEmulatorLaunchers "ryujinx" -} \ No newline at end of file +} From 3d83f2c1c9e7cc56274f5c3d7671409a432cc8a7 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 18 Dec 2024 13:59:13 +0100 Subject: [PATCH 158/230] change Symlinks to the emulator folder, not on the savePath --- functions/helperFunctions.sh | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/functions/helperFunctions.sh b/functions/helperFunctions.sh index 334c05e89..329a95428 100644 --- a/functions/helperFunctions.sh +++ b/functions/helperFunctions.sh @@ -428,30 +428,26 @@ function getReleaseURLGH(){ } function linkToSaveFolder(){ - local emu=$1 - local folderName=$2 - local path=$3 + local emu=$1 # /saves/$emu/ + local folderName=$2 # states or saves + local path=$3 # original location - if [ ! -d "$savesPath/$emu/$folderName" ]; then - if [ ! -L "$savesPath/$emu/$folderName" ]; then - mkdir -p "$savesPath/$emu" + mkdir -p "$path" + + # Es el path un simlink? + if [ ! -L "$path" ]; then + #We delete the old symlink and move the saves + if [ -L "$savesPath/$emu/$folderName" ]; then setMSG "Linking $emu $folderName to the Emulation/saves folder" - mkdir -p "$path" - ln -snv "$path" "$savesPath/$emu/$folderName" + rm -rf "$savesPath/$emu/$folderName" + mv "$path" "$savesPath/$emu/$folderName" + ln -snv "$savesPath/$emu/$folderName" "$path" fi else - if [ ! -L "$savesPath/$emu/$folderName" ]; then - echo "$savesPath/$emu/$folderName is not a link. Please check it." - else - if [ $(readlink $savesPath/$emu/$folderName) == $path ]; then - echo "$savesPath/$emu/$folderName is already linked." - echo " Target: $(readlink $savesPath/$emu/$folderName)" - else - echo "$savesPath/$emu/$folderName not linked correctly." - unlink "$savesPath/$emu/$folderName" - linkToSaveFolder "$emu" "$folderName" "$path" - fi - fi + setMSG "Rebuilding Link $emu $folderName to the Emulation/saves folder" + mkdir -p "$savesPath/$emu/$folderName" + unlink "$savesPath/$emu/$folderName" + ln -snv "$savesPath/$emu/$folderName" "$path" fi } From 6a6ded6d93b6f75bf722d3acc259f929df914eaa Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 19 Dec 2024 11:10:22 +0100 Subject: [PATCH 159/230] Revert "Update emuDeckRyujinx.sh" This reverts commit fb6c0d541b261aabb88317c628a47ff226a3d49b. --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index e52e32426..4cbfcafe9 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "ryujinx-mirror/ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" From 72452252616dbe6ce818f77752cb86eab53e8a78 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 09:16:39 +0100 Subject: [PATCH 160/230] no individual artwork testing --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index fc16aa462..5faf69f38 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -42,7 +42,7 @@ generateGameLists_artwork() { echo "Searching for missing artwork" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" - $(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & + #$(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & echo "Artwork finished. Restart if you see this message" > "$MSG" } From e39349dff034abc2a6b725e0139584b2e07bd01d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 09:33:51 +0100 Subject: [PATCH 161/230] symlink fixes --- functions/generateGameLists.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 5faf69f38..088c6cb17 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -7,9 +7,10 @@ generateGameLists() { local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Starting to build database" > "$MSG" mkdir -p "$storagePath/retrolibrary/artwork" - mkdir -p "$accountfolder/config/grid/retrolibrary/" - ln -s "$storagePath/retrolibrary/artwork/" "$accountfolder/config/grid/retrolibrary/artwork" - ln -s "$storagePath/retrolibrary/cache/" "$accountfolder/config/grid/retrolibrary/cache" + mkdir -p "$accountfolder/config/grid/retrolibrary/cache" + + ln -sf "$storagePath/retrolibrary/artwork" "$accountfolder/config/grid/retrolibrary/artwork" + ln -sf "$storagePath/retrolibrary/cache" "$accountfolder/config/grid/retrolibrary/cache" generateGameLists_downloadAchievements generateGameLists_downloadData From 3d13a88e68e12d095e2384f30afcfa424e7ab134 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 09:58:33 +0100 Subject: [PATCH 162/230] $storagePath --- functions/generateGameLists.sh | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 088c6cb17..7d2aaaef1 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -7,16 +7,21 @@ generateGameLists() { local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Starting to build database" > "$MSG" mkdir -p "$storagePath/retrolibrary/artwork" - mkdir -p "$accountfolder/config/grid/retrolibrary/cache" + mkdir -p "$storagePath/retrolibrary/cache" + mkdir -p "$storagePath/retrolibrary/data" + mkdir -p "$storagePath/retrolibrary/achievements" + mkdir -p "$accountfolder/config/grid/retrolibrary/" ln -sf "$storagePath/retrolibrary/artwork" "$accountfolder/config/grid/retrolibrary/artwork" ln -sf "$storagePath/retrolibrary/cache" "$accountfolder/config/grid/retrolibrary/cache" + ln -sf "$storagePath/retrolibrary/data" "$accountfolder/config/grid/retrolibrary/data" + ln -sf "$storagePath/retrolibrary/achievements" "$accountfolder/config/grid/retrolibrary/achievements" generateGameLists_downloadAchievements generateGameLists_downloadData pegasus_setPaths - rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$dest_folder" --keep-dirlinks + rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$storagePath/retrolibrary/artwork" --keep-dirlinks mkdir -p "$HOME/emudeck/cache/" echo "Database built" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" @@ -37,11 +42,8 @@ generateGameLists_importESDE() { } generateGameLists_artwork() { - mkdir -p "$HOME/emudeck/cache/" - local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Searching for missing artwork" > "$MSG" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$storagePath/retrolibrary/artwork" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$storagePath/retrolibrary/artwork" #$(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & echo "Artwork finished. Restart if you see this message" > "$MSG" @@ -52,7 +54,7 @@ saveImage(){ local name=$2 local system=$3 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/${system}/media/box2dfront/" + local dest_folder="$storagePath/retrolibrary/artwork/${system}/media/box2dfront/" local dest_path="$dest_folder/$name.jpg" wget -q -O "$dest_path" "$url" } @@ -67,7 +69,7 @@ function addGameListsArtwork() { #local tempGrid=$(generateGameLists_extraArtwork $file $platform) #local grid=$(echo "$tempGrid" | jq -r '.grid') - local vertical="$accountfolder/config/grid/retrolibrary/artwork/$platform/media/box2dfront/$file.jpg" + local vertical="$storagePath/retrolibrary/artwork/$platform/media/box2dfront/$file.jpg" local grid=$vertical local destination_vertical="$accountfolder/config/grid/${appID}p.png" #vertical local destination_hero="$accountfolder/config/grid/${appID}_hero.png" #BG @@ -77,20 +79,20 @@ function addGameListsArtwork() { rm -rf "$destination_grid" #Use CP if custom grid instead of ln.. - ln -s "$vertical" "$destination_vertical" - ln -s "$grid" "$destination_hero" - ln -s "$grid" "$destination_grid" + ln -sf "$vertical" "$destination_vertical" + ln -sf "$grid" "$destination_hero" + ln -sf "$grid" "$destination_grid" } generateGameLists_getPercentage() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" + local dest_folder="$storagePath/retrolibrary/artwork/" - python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" - local json_file="$HOME/emudeck/cache/roms_games.json" - local json_file_artwork="$HOME/emudeck/cache/missing_artwork.json" + local json_file="$storagePath/retrolibrary/cache/roms_games.json" + local json_file_artwork="$storagePath/retrolibrary/cache/missing_artwork.json" # Contar el número total de juegos en `roms_games.json` local games=$(jq '[.[].games[]] | length' "$json_file") @@ -112,7 +114,7 @@ generateGameLists_extraArtwork() { local platform=$2 local hash=$3 local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) - local dest_folder="$accountfolder/config/grid/retrolibrary/artwork" + local dest_folder="$storagePath/retrolibrary/artwork" wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game&hash=$hash" From 88ebb7958267917c91a9b621427a512454ca15d5 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:00:10 +0100 Subject: [PATCH 163/230] cache folder --- functions/generateGameLists.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 7d2aaaef1..35ef62726 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -22,7 +22,7 @@ generateGameLists() { pegasus_setPaths rsync -r --exclude='roms' --exclude='txt' "$EMUDECKGIT/roms/" "$storagePath/retrolibrary/artwork" --keep-dirlinks - mkdir -p "$HOME/emudeck/cache/" + mkdir -p "$storagePath/retrolibrary/cache/" echo "Database built" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" } @@ -31,7 +31,7 @@ generateGameListsJson() { echo "Adding Games" > "$MSG" #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" echo "Games Added" > "$MSG" - #cat $HOME/emudeck/cache/roms_games.json + #cat $storagePath/retrolibrary/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & generateGameLists_artwork &> /dev/null & @@ -116,10 +116,10 @@ generateGameLists_extraArtwork() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$storagePath/retrolibrary/artwork" - wget -q -O "$HOME/emudeck/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game&hash=$hash" + wget -q -O "$storagePath/retrolibrary/cache/response.json" "https://bot.emudeck.com/steamdb_extra.php?name=$game&hash=$hash" - game_name=$(jq -r '.name' "$HOME/emudeck/cache/response.json") - game_img_url=$(jq -r '.grid' "$HOME/emudeck/cache/response.json") + game_name=$(jq -r '.name' "$storagePath/retrolibrary/cache/response.json") + game_img_url=$(jq -r '.grid' "$storagePath/retrolibrary/cache/response.json") dest_path="$dest_folder/$platform/$game.grid.temp" if [ "$game_img_url" != "null" ]; then From 24e045ee9c4c4ecf848266b4fc6dcf2eb0579faf Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:03:08 +0100 Subject: [PATCH 164/230] -avR --- functions/ToolScripts/emuDeckPegasus.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/ToolScripts/emuDeckPegasus.sh b/functions/ToolScripts/emuDeckPegasus.sh index 7383040d4..b3d645364 100644 --- a/functions/ToolScripts/emuDeckPegasus.sh +++ b/functions/ToolScripts/emuDeckPegasus.sh @@ -34,8 +34,8 @@ pegasus_install(){ } pegasus_setPaths(){ - rsync -r --exclude='roms' --exclude='pfx' "$EMUDECKGIT/roms/" "$romsPath" --keep-dirlinks - rsync -av --exclude='roms' --exclude='pfx' "$EMUDECKGIT/roms/" "$toolsPath/downloaded_media" + rsync -avR --exclude='roms' --exclude='pfx' "$EMUDECKGIT/roms/" "$romsPath" --keep-dirlinks + rsync -avR --exclude='roms' --exclude='pfx' "$EMUDECKGIT/roms/" "$toolsPath/downloaded_media" find $romsPath/ -type f -name "metadata.txt" -exec sed -i "s|CORESPATH|${RetroArch_cores}|g" {} \; find $romsPath/ -type f -name "metadata.txt" -exec sed -i "s|/run/media/mmcblk0p1/Emulation|${emulationPath}|g" {} \; From 95e71640247bea5510dd56d5d04d182e3c92efef Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:06:14 +0100 Subject: [PATCH 165/230] no hash --- tools/retro-library/generate_game_lists.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 1bd8a59ed..7ee0dd495 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -134,15 +134,14 @@ def collect_game_data(system_dir, extensions): name_cleaned_pegasus = name.replace(',_', ',') name_cleaned = name_cleaned.lower() # Calculate the ROM hash - rom_hash = calculate_hash(file_path) + #rom_hash = calculate_hash(file_path) game_info = { "name": name_cleaned, "filename": file_path, "file": name_cleaned, "img": f"/customimages/retrolibrary/artwork/{platform}/media", - "platform": platform, - "hash": rom_hash + "platform": platform } game_data.append(game_info) return sorted(game_data, key=lambda x: x['name']) From 83699fc16f739af5c1f31d568e3066924653bc9a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:09:26 +0100 Subject: [PATCH 166/230] storage_path --- tools/retro-library/missing_artwork_platforms.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 0aa5127b8..e155f1ac0 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -40,6 +40,8 @@ def getSettings(): return configuration +settings = getSettings() +storage_path = os.path.expandvars(settings["storagePath"]) # Función para escribir en el archivo de log def log_message(message): From 7f27584b786c0ee7218f6b7952c64528e52c1bef Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:09:42 +0100 Subject: [PATCH 167/230] Revert "no hash" This reverts commit 95e71640247bea5510dd56d5d04d182e3c92efef. --- tools/retro-library/generate_game_lists.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 7ee0dd495..1bd8a59ed 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -134,14 +134,15 @@ def collect_game_data(system_dir, extensions): name_cleaned_pegasus = name.replace(',_', ',') name_cleaned = name_cleaned.lower() # Calculate the ROM hash - #rom_hash = calculate_hash(file_path) + rom_hash = calculate_hash(file_path) game_info = { "name": name_cleaned, "filename": file_path, "file": name_cleaned, "img": f"/customimages/retrolibrary/artwork/{platform}/media", - "platform": platform + "platform": platform, + "hash": rom_hash } game_data.append(game_info) return sorted(game_data, key=lambda x: x['name']) From 3a39dc8fe99f087d6058ea8eddc6c574056abbee Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:17:20 +0100 Subject: [PATCH 168/230] generateGameLists_pythonEnv --- functions/generateGameLists.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 35ef62726..3220dfc3b 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -2,7 +2,20 @@ MSG="$HOME/.config/EmuDeck/msg.log" +generateGameLists_pythonEnv() { + if [ ! -d "$storagePath/retrolibrary/python_virtual_env" ]; then + python3 -m venv "$storagePath/retrolibrary/python_virtual_env" + source "$storagePath/retrolibrary/python_virtual_env/bin/activate" + pip install requests + else + source "$storagePath/retrolibrary/python_virtual_env/bin/activate" + fi +} + generateGameLists() { + + generateGameLists_pythonEnv + local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" echo "Starting to build database" > "$MSG" @@ -28,6 +41,7 @@ generateGameLists() { } generateGameListsJson() { + generateGameLists_pythonEnv echo "Adding Games" > "$MSG" #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" echo "Games Added" > "$MSG" From 67dbfdd0ea0374d313e5a287d96f981e8fd6502c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:24:10 +0100 Subject: [PATCH 169/230] python folder --- functions/generateGameLists.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 3220dfc3b..c5786c397 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -3,12 +3,12 @@ MSG="$HOME/.config/EmuDeck/msg.log" generateGameLists_pythonEnv() { - if [ ! -d "$storagePath/retrolibrary/python_virtual_env" ]; then - python3 -m venv "$storagePath/retrolibrary/python_virtual_env" - source "$storagePath/retrolibrary/python_virtual_env/bin/activate" + if [ ! -d "$HOME/.config/EmuDeck/python_virtual_env" ]; then + python3 -m venv "$HOME/.config/EmuDeck/python_virtual_env" + source "$HOME/.config/EmuDeck/python_virtual_env/bin/activate" pip install requests else - source "$storagePath/retrolibrary/python_virtual_env/bin/activate" + source "$HOME/.config/EmuDeck/python_virtual_env/bin/activate" fi } From 4e1b78cba3f8cbca243933412d9df371b87b3507 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:26:06 +0100 Subject: [PATCH 170/230] no hash --- tools/retro-library/generate_game_lists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 1bd8a59ed..c81f0749b 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -134,7 +134,7 @@ def collect_game_data(system_dir, extensions): name_cleaned_pegasus = name.replace(',_', ',') name_cleaned = name_cleaned.lower() # Calculate the ROM hash - rom_hash = calculate_hash(file_path) + rom_hash = "000" game_info = { "name": name_cleaned, From 4213045e0be75edc6077555bbbc2fe494385a802 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:27:56 +0100 Subject: [PATCH 171/230] cat --- functions/generateGameLists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index c5786c397..995bd4826 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -45,7 +45,7 @@ generateGameListsJson() { echo "Adding Games" > "$MSG" #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" echo "Games Added" > "$MSG" - #cat $storagePath/retrolibrary/cache/roms_games.json + cat $storagePath/retrolibrary/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & generateGameLists_artwork &> /dev/null & From 3e81914da49ee7d0ff47a24d4d547e90f9fda6f4 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:30:36 +0100 Subject: [PATCH 172/230] jotasones --- functions/generateGameLists.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 995bd4826..30146a55e 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -38,6 +38,7 @@ generateGameLists() { mkdir -p "$storagePath/retrolibrary/cache/" echo "Database built" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" + generateGameLists_artwork &> /dev/null & } generateGameListsJson() { @@ -47,7 +48,7 @@ generateGameListsJson() { echo "Games Added" > "$MSG" cat $storagePath/retrolibrary/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & - generateGameLists_artwork &> /dev/null & + #generateGameLists_artwork &> /dev/null & } From 24c5ed1cf25097cd4677c78bbd6c58af3924b56e Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:31:10 +0100 Subject: [PATCH 173/230] no env --- functions/generateGameLists.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 30146a55e..e1348eaae 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -42,14 +42,12 @@ generateGameLists() { } generateGameListsJson() { - generateGameLists_pythonEnv echo "Adding Games" > "$MSG" #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" echo "Games Added" > "$MSG" cat $storagePath/retrolibrary/cache/roms_games.json #generateGameLists_artwork $userid &> /dev/null & #generateGameLists_artwork &> /dev/null & - } generateGameLists_importESDE() { From 6f8ea417fcc1d397712be71da12160b5139c6bac Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:39:59 +0100 Subject: [PATCH 174/230] env --- functions/generateGameLists.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index e1348eaae..1a53d1584 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -14,7 +14,7 @@ generateGameLists_pythonEnv() { generateGameLists() { - generateGameLists_pythonEnv + generateGameLists_pythonEnv &> /dev/null local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" @@ -42,6 +42,7 @@ generateGameLists() { } generateGameListsJson() { + generateGameLists_pythonEnv &> /dev/null echo "Adding Games" > "$MSG" #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" echo "Games Added" > "$MSG" @@ -51,10 +52,12 @@ generateGameListsJson() { } generateGameLists_importESDE() { + generateGameLists_pythonEnv &> /dev/null python $HOME/.config/EmuDeck/backend/tools/retro-library/import_media.py "$romsPath" "$dest_folder" } generateGameLists_artwork() { + generateGameLists_pythonEnv &> /dev/null echo "Searching for missing artwork" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$storagePath/retrolibrary/artwork" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$storagePath/retrolibrary/artwork" @@ -63,6 +66,7 @@ generateGameLists_artwork() { } saveImage(){ + generateGameLists_pythonEnv &> /dev/null local url=$1 local name=$2 local system=$3 @@ -73,6 +77,7 @@ saveImage(){ } function addGameListsArtwork() { + generateGameLists_pythonEnv &> /dev/null local file="$1" local appID="$2" local platform="$3" @@ -98,7 +103,7 @@ function addGameListsArtwork() { } generateGameLists_getPercentage() { - + generateGameLists_pythonEnv &> /dev/null local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$storagePath/retrolibrary/artwork/" @@ -123,6 +128,7 @@ generateGameLists_getPercentage() { } generateGameLists_extraArtwork() { + generateGameLists_pythonEnv &> /dev/null local game=$1 local platform=$2 local hash=$3 @@ -144,6 +150,7 @@ generateGameLists_extraArtwork() { } generateGameLists_retroAchievements(){ + generateGameLists_pythonEnv &> /dev/null local hash=$1 local system=$2 local localDataPath="$storagePath/retrolibrary/achievements/$system.json" @@ -151,6 +158,7 @@ generateGameLists_retroAchievements(){ } generateGameLists_downloadAchievements(){ + generateGameLists_pythonEnv &> /dev/null local folder="$storagePath/retrolibrary/achievements" if [ ! -d $folder ]; then echo "Downloading Retroachievements Data" > "$MSG" @@ -162,6 +170,7 @@ generateGameLists_downloadAchievements(){ } generateGameLists_downloadData(){ + generateGameLists_pythonEnv &> /dev/null local folder="$storagePath/retrolibrary/data" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) ln -s "$folder" "$accountfolder/config/grid/retrolibrary/data" From 2203c23bb02f4f058691ae2f6159bcd21fa63a07 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 10:42:49 +0100 Subject: [PATCH 175/230] fix --- functions/generateGameLists.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 1a53d1584..d472a891e 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -21,14 +21,10 @@ generateGameLists() { echo "Starting to build database" > "$MSG" mkdir -p "$storagePath/retrolibrary/artwork" mkdir -p "$storagePath/retrolibrary/cache" - mkdir -p "$storagePath/retrolibrary/data" - mkdir -p "$storagePath/retrolibrary/achievements" mkdir -p "$accountfolder/config/grid/retrolibrary/" ln -sf "$storagePath/retrolibrary/artwork" "$accountfolder/config/grid/retrolibrary/artwork" ln -sf "$storagePath/retrolibrary/cache" "$accountfolder/config/grid/retrolibrary/cache" - ln -sf "$storagePath/retrolibrary/data" "$accountfolder/config/grid/retrolibrary/data" - ln -sf "$storagePath/retrolibrary/achievements" "$accountfolder/config/grid/retrolibrary/achievements" generateGameLists_downloadAchievements generateGameLists_downloadData @@ -163,6 +159,7 @@ generateGameLists_downloadAchievements(){ if [ ! -d $folder ]; then echo "Downloading Retroachievements Data" > "$MSG" mkdir -p $folder + ln -sf "$storagePath/retrolibrary/achievements" "$accountfolder/config/grid/retrolibrary/achievements" wget -q -O "$folder/achievements.zip" "https://bot.emudeck.com/achievements/achievements.zip" cd "$folder" && unzip -o achievements.zip && rm achievements.zip echo "Retroachievements Data Downloaded" > "$MSG" @@ -177,6 +174,7 @@ generateGameLists_downloadData(){ if [ ! -d $folder ]; then echo "Downloading Metada" > "$MSG" mkdir -p $folder + ln -sf "$storagePath/retrolibrary/data" "$accountfolder/config/grid/retrolibrary/data" wget -q -O "$folder/data.zip" "https://bot.emudeck.com/data/data.zip" cd $folder && unzip -o data.zip && rm data.zip echo "Metada Downloaded" > "$MSG" From 4904d4d8b98d91b6d64fdd18fc561866c0ab9f17 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 11:34:32 +0100 Subject: [PATCH 176/230] screenscraper artwork --- functions/generateGameLists.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index d472a891e..06f9cef3c 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -57,12 +57,11 @@ generateGameLists_artwork() { echo "Searching for missing artwork" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$storagePath/retrolibrary/artwork" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$storagePath/retrolibrary/artwork" - #$(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$dest_folder") & + $(python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$storagePath/retrolibrary/artwork" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art.py "$storagePath/retrolibrary/artwork") & echo "Artwork finished. Restart if you see this message" > "$MSG" } saveImage(){ - generateGameLists_pythonEnv &> /dev/null local url=$1 local name=$2 local system=$3 @@ -73,7 +72,6 @@ saveImage(){ } function addGameListsArtwork() { - generateGameLists_pythonEnv &> /dev/null local file="$1" local appID="$2" local platform="$3" @@ -124,7 +122,6 @@ generateGameLists_getPercentage() { } generateGameLists_extraArtwork() { - generateGameLists_pythonEnv &> /dev/null local game=$1 local platform=$2 local hash=$3 @@ -154,7 +151,6 @@ generateGameLists_retroAchievements(){ } generateGameLists_downloadAchievements(){ - generateGameLists_pythonEnv &> /dev/null local folder="$storagePath/retrolibrary/achievements" if [ ! -d $folder ]; then echo "Downloading Retroachievements Data" > "$MSG" @@ -167,7 +163,6 @@ generateGameLists_downloadAchievements(){ } generateGameLists_downloadData(){ - generateGameLists_pythonEnv &> /dev/null local folder="$storagePath/retrolibrary/data" local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) ln -s "$folder" "$accountfolder/config/grid/retrolibrary/data" From f80cb9b06046e6e84d483bd87405e46a7047d6dd Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 11:47:42 +0100 Subject: [PATCH 177/230] no hash, too slow --- tools/retro-library/generate_game_lists.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index c81f0749b..0c8f9a5cb 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -2,7 +2,6 @@ import json import sys import re -import hashlib import subprocess # Define the log file path @@ -92,17 +91,6 @@ def clean_name(filename): log_message(f"Saved states JSON written to {output_file}") def generate_game_lists(roms_path): - def calculate_hash(file_path): - """Calculate the MD5 hash of a file.""" - hash_md5 = hashlib.md5() - try: - with open(file_path, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - except Exception as e: - return None - def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -133,16 +121,14 @@ def collect_game_data(system_dir, extensions): name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') name_cleaned_pegasus = name.replace(',_', ',') name_cleaned = name_cleaned.lower() - # Calculate the ROM hash - rom_hash = "000" + game_info = { "name": name_cleaned, "filename": file_path, "file": name_cleaned, "img": f"/customimages/retrolibrary/artwork/{platform}/media", - "platform": platform, - "hash": rom_hash + "platform": platform } game_data.append(game_info) return sorted(game_data, key=lambda x: x['name']) From 7831411f257371dbb7724e83170ddf0402a84b26 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 11:49:14 +0100 Subject: [PATCH 178/230] faster hash --- tools/retro-library/missing_artwork.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 6d2780a56..350972965 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -48,6 +48,7 @@ def getSettings(): # Function to write messages to the log file def log_message(message): + print(message) with open(msg_file, "w") as log_file: # "a" to append messages without overwriting log_file.write(message + "\n") @@ -56,10 +57,11 @@ def calculate_hash(file_path): hash_md5 = hashlib.md5() try: with open(file_path, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) + # Mapea el archivo completo a memoria + with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: + hash_md5.update(mm) return hash_md5.hexdigest() - except Exception as e: + except Exception: return None def collect_game_data(system_dir, extensions): From 9ec7803e7942033e92aaeb3f9c32f97079fe9e67 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 11:56:56 +0100 Subject: [PATCH 179/230] working hash --- tools/retro-library/missing_artwork.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 350972965..78b503228 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -54,16 +54,16 @@ def log_message(message): def generate_game_lists(roms_path, images_path): def calculate_hash(file_path): + import hashlib hash_md5 = hashlib.md5() try: with open(file_path, "rb") as f: - # Mapea el archivo completo a memoria - with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: - hash_md5.update(mm) + for chunk in iter(lambda: f.read(65536), b""): # 64 KB chunks + hash_md5.update(chunk) + print(hash_md5.hexdigest()) return hash_md5.hexdigest() except Exception: - return None - + return None def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): From d98ed3ad1a2e77e488ec1d620c08edef221693a6 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 12:22:20 +0100 Subject: [PATCH 180/230] model2 --- tools/retro-library/generate_game_lists.py | 2 ++ tools/retro-library/missing_artwork_platforms.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index 0c8f9a5cb..e6337c68b 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -139,6 +139,8 @@ def collect_game_data(system_dir, extensions): for system_dir in os.listdir(roms_dir): if system_dir == "xbox360": system_dir = "xbox360/roms" + if system_dir == "model2": + system_dir = "model2/roms" full_path = os.path.join(roms_dir, system_dir) if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index e155f1ac0..5c9552578 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -79,7 +79,9 @@ def has_missing_images(system_dir, extensions): for system_dir in os.listdir(roms_dir): if system_dir == "xbox360": - system_dir = "xbox360/roms" + system_dir = "xbox360/roms" + if system_dir == "model2": + system_dir = "model2/roms" full_path = os.path.join(roms_dir, system_dir) if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) From 56e01a46a7b1231e7aeaa936b589970c15843e21 Mon Sep 17 00:00:00 2001 From: "Loading.oO" Date: Mon, 23 Dec 2024 22:14:03 +0000 Subject: [PATCH 181/230] Added ShadPS4 Flatpak (#1357) * Adding ShadPS4 Flatpak * Amended Flatpak installation steps for ShadPS4 * Setting input config, optimize settings management and language support * Additional amendments for consistency --- .../config/shadps4/config.toml | 68 ++++ .../config/shadps4/inputConfig/default.ini | 67 ++++ .../steam-deck-romate/userConfigurations.json | 16 + functions/EmuScripts/emuDeckShadPS4.sh | 372 ++++++++++++++++++ functions/ToolScripts/emuDeckMigration.sh | 2 + functions/all.sh | 1 + functions/helperFunctions.sh | 1 + functions/jsonToBashVars.sh | 1 + settings.sh | 1 + setup.sh | 4 + uninstall.sh | 10 + versions.json | 1 + 12 files changed, 544 insertions(+) create mode 100644 configs/net.shadps4.ShadPS4/config/shadps4/config.toml create mode 100644 configs/net.shadps4.ShadPS4/config/shadps4/inputConfig/default.ini create mode 100644 functions/EmuScripts/emuDeckShadPS4.sh diff --git a/configs/net.shadps4.ShadPS4/config/shadps4/config.toml b/configs/net.shadps4.ShadPS4/config/shadps4/config.toml new file mode 100644 index 000000000..8a048a0cc --- /dev/null +++ b/configs/net.shadps4.ShadPS4/config/shadps4/config.toml @@ -0,0 +1,68 @@ +# ShadPS4 Settings/Config file + +[General] +isPS4Pro = false +Fullscreen = false +playBGM = false +BGMvolume = 50 +enableDiscordRPC = false +logType = "async" # Configures logging synchronization (sync/async) +logFilter = "" # Sets the logging category for various logging classes +userName = "shadPS4" +updateChannel = "Release" +showSplash = false +autoUpdate = false +separateUpdateEnabled = false + +[Input] +cursorState = 0 # HideCursorState::Idle +cursorHideTimeout = 5 +backButtonBehavior = "left" +useSpecialPad = false +specialPadClass = 1 + +[GPU] +# Configures the game window width and height e.g. '800' for OLED Steameck (16:10 Aspect Ratio) +screenWidth = 1280 +screenHeight = 720 +nullGpu = false # Disables rendering +copyGPUBuffers = false +dumpShaders = false # Dump shaders that are loaded by the emulator. Dump path: ../user/shader/dumps +patchShaders = true +vblankDivider = 1 + +[Vulkan] +gpuId = -1 +validation = false +validation_sync = false +validation_gpu = false +rdocEnable = false # Automatically hook RenderDoc when installed. Useful for debugging shaders and game rendering +rdocMarkersEnable = false # Enable automatic RenderDoc event annotation +crashDiagnostic = false + +[Debug] +DebugDump = false +CollectShader = false + +[GUI] +theme = 0 +iconSize = 36 +iconSizeGrid = 69 +sliderPos = 0 +sliderPosGrid = 0 +gameTableMode = 0 +mw_width = 1280 +mw_height = 720 # Need to set to '800' for OLED Deck (16:10 Aspect Ratio) +geometry_x = 400 +geometry_y = 400 +geometry_w = 1280 +geometry_h = 720 # Need to set to '800' for OLED Deck (16:10 Aspect Ratio) +installDirs = [] +addonInstallDir = "" +pkgDirs = [] +elfDirs = [] +recentFiles = [] +emulatorLanguage = "en" + +[Settings] +consoleLanguage = 1 # english diff --git a/configs/net.shadps4.ShadPS4/config/shadps4/inputConfig/default.ini b/configs/net.shadps4.ShadPS4/config/shadps4/inputConfig/default.ini new file mode 100644 index 000000000..0ba55e322 --- /dev/null +++ b/configs/net.shadps4.ShadPS4/config/shadps4/inputConfig/default.ini @@ -0,0 +1,67 @@ +# Default input configuration file, each game may have per-game files too, e.g. + ;~/config/shadps4/ + ;└── inputConfig/ + ; ├── default.ini + ; └── [game_id].ini + +# Keyboard bindings +triangle = f +circle = space +cross = e +square = r + +pad_up = w, lalt +pad_up = mousewheelup +pad_down = s, lalt +pad_down = mousewheeldown +pad_left = a, lalt +pad_left = mousewheelleft +pad_right = d, lalt +pad_right = mousewheelright + +l1 = rightbutton, lshift +r1 = leftbutton +l2 = rightbutton +r2 = leftbutton, lshift +l3 = x +r3 = q +r3 = middlebutton + +options = escape +touchpad = g + +key_toggle = i, lalt +mouse_to_joystick = right +mouse_movement_params = 0.5, 1, 0.125 +leftjoystick_halfmode = lctrl + +axis_left_x_minus = a +axis_left_x_plus = d +axis_left_y_minus = w +axis_left_y_plus = s + +# Controller bindings +triangle = triangle +cross = cross +square = square +circle = circle + +l1 = l1 +l2 = l2 +l3 = l3 +r1 = r1 +r2 = r2 +r3 = r3 + +pad_up = pad_up +pad_down = pad_down +pad_left = pad_left +pad_right = pad_right + +options = options +touchpad = back + +axis_left_x = axis_left_x +axis_left_y = axis_left_y +axis_right_x = axis_right_x +axis_right_y = axis_right_y diff --git a/configs/steam-deck-romate/userConfigurations.json b/configs/steam-deck-romate/userConfigurations.json index ae200386b..699d7a253 100644 --- a/configs/steam-deck-romate/userConfigurations.json +++ b/configs/steam-deck-romate/userConfigurations.json @@ -1203,6 +1203,22 @@ "category":"PlayStation 3", "name":"PlayStation 3" }, + { + "id": "12345678901234567", + "executable": { + "path": "\"/usr/bin/flatpak\" run net.shadps4.shadPS4 --no-gui \"${filePath}\"", + "arguments": "" + }, + "romDirectory": "roms/ps4", + "supportedFileTypes": [ + "iso", + "ISO", + "pkg", + "PKG" + ], + "category": "PlayStation 4", + "name": "PlayStation 4" + }, { "id":"164785656634747068", "executable":{ diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh new file mode 100644 index 000000000..26d340184 --- /dev/null +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -0,0 +1,372 @@ +#!/bin/bash + +# Script to install, initialize and configure ShadPS4 on EmuDeck +# Note: No Bios/Keys symlinks necessary + +# External helper functions (defined outside this script) +#- installEmuBI() +#- getReleaseURLGH() +#- configEmuAI() +#- linkToSaveFolder() +#- uninstallGeneric() +#- migrateAndLinkConfig() +#- flushEmulatorLaunchers() +#- setMSG() + +# Variables +ShadPS4_emuName="ShadPS4" +ShadPS4_emuType="$emuDeckEmuTypeBinary" +ShadPS4_emuPath="$HOME/Applications/publish" +ShadPS4_configFile="$HOME/.config/shadps4/config.toml" +userDir="$HOME/.config/shadps4/user" +sysDir="$HOME/.config/shadps4/system" +inputConfigDir="$HOME/.config/shadps4/inputConfig" +controllerFile="${inputConfigDir}/default.ini" + +migrationFlag="$HOME/.config/EmuDeck/.${ShadPS4_emuName}MigrationCompleted" + +# Language keys using [ISO 639-1: Language codes] & [ISO 3166-1 alpha-2: Country codes] +# NOTE: Keep in sync with https://github.com/shadps4-emu/shadPS4/tree/main/src/qt_gui/translations +# even though project still just uses some two character codes e.g. 'nl' instead of 'nl_NL' +declare -A ShadPS4_languages +ShadPS4_languages=( + ["ar"]="Arabic" + ["da_DK"]="Danish" + ["de"]="German Deutsch" + ["el"]="Greek" + ["el_GR"]="Greek" + ["en"]="English" + ["en_US"]="English (US)" + ["en_IE"]="English (Irish)" + ["es_ES"]="Spanish" + ["fa_IR"]="Farsi (Iran)" + ["fi"]="Finnish" + ["fi_FI"]="Finnish" + ["fr"]="French" + ["fr_FR"]="French" + ["hu_HU"]="Hungarian" + ["id"]="Indonesian" + ["it"]="Italian" + ["ja_JP"]="Japanese" + ["ko_KR"]="Korean" + ["lt_LT"]="Lithuanian" + ["nb"]="Norwegian Bokmål" + ["nl"]="Dutch" + ["nl_NL"]="Dutch (Netherlands)" + ["pl_PL"]="Polish" + ["pt_BR"]="Portuguese" + ["ro_RO"]="Romanian" + ["ru_RU"]="Russian" + ["sq"]="Albanian" + ["ti_ER"]="Tigrinya" + ["tr_TR"]="Turkish" + ["uk_UA"]="Ukrainian" + ["vi_VN"]="Vietnamese" + ["zh_CN"]="Chinese (Simplified)" + ["zh_TW"]="Traditional Chinese (Taiwan)" +) + +declare -A ShadPS4_regions +ShadPS4_regions=( + ["ar"]="Arabic" + ["da_DK"]="Denmark" + ["de"]="Deutsch" + ["el"]="Greece" + ["el_GR"]="Greece" + ["en"]="Global English" + ["en_US"]="United States" + ["en_IE"]="Ireland" + ["es_ES"]="Spain" + ["fa_IR"]="Iran" + ["fi"]="Finland" + ["fi_FI"]="Finland" + ["fr"]="France" + ["fr_FR"]="France" + ["hu_HU"]="Hungary" + ["id"]="Indonesia" + ["it"]="Italian" + ["ja_JP"]="Japan" + ["ko_KR"]="South Korea" + ["lt_LT"]="Lithuania" + ["nb"]="Norway" + ["nl"]="Netherlands" + ["nl_NL"]="Netherlands" + ["pl_PL"]="Poland" + ["pt_BR"]="Brazil" + ["ro_RO"]="Romania" + ["ru_RU"]="Russia" + ["sq"]="Albania" + ["ti_ER"]="Eritrea" + ["tr_TR"]="Turkey" + ["uk_UA"]="Ukraine" + ["vi_VN"]="Vietnam" + ["zh_CN"]="China" + ["zh_TW"]="Taiwan" +) + +ShadPS4_cleanup(){ + echo "Begin ShadPS4 Cleanup" +} + +# TODO: Install Flatpak from https://github.com/shadps4-emu/shadPS4-flatpak +ShadPS4_install(){ + echo "Begin ShadPS4 Install" + local showProgress=$1 + + if installEmuBI "$ShadPS4_emuName" "$(getReleaseURLGH "ShadPS4/shadps4" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + mkdir -p "$HOME/Applications/publish" + tar -xvf "$HOME/Applications/shadps4.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/shadps4.tar.gz" + chmod +x "$HOME/Applications/publish/shadps4" + else + return 1 + fi + + # Flatpak install + echo "Installing ShadPS4 via Flatpak..." + flatpak install flathub net.shadps4.shadPS4 -y --user + + # Move Flatpak installed files to the desired location + mkdir -p "$HOME/Applications/publish" + rsync -av "$HOME/.local/share/flatpak/app/net.shadps4.shadPS4/x86_64/stable/active/files/bin/" "$HOME/Applications/publish/" && flatpak uninstall flathub net.shadps4.shadPS4 -y --user + + # Clean up old games directory if it exists + rm -rf "$HOME/.config/shadps4/games" + + # Set executable permission + chmod +x "$HOME/Applications/publish/shadps4" +} + +ShadPS4_init(){ + configEmuAI "$ShadPS4_emuName" "config" "$HOME/.config/shadps4" "$EMUDECKGIT/configs/shadps4" "true" + ShadPS4_setupStorage + ShadPS4_setEmulationFolder + ShadPS4_setupSaves + ShadPS4_flushEmulatorLauncher + ShadPS4_setLanguage + + # SRM_createParsers + # ShadPS4_migrate +} + +ShadPS4_update(){ + echo "Begin ShadPS4 update" + + configEmuAI "$ShadPS4_emuName" "config" "$HOME/.config/shadps4" "$EMUDECKGIT/configs/shadps4" + + ShadPS4_setEmulationFolder + ShadPS4_setupStorage + ShadPS4_setupSaves + ShadPS4_finalize + ShadPS4_flushEmulatorLauncher +} + +# Configuration Paths +ShadPS4_setEmulationFolder(){ + echo "Begin ShadPS4 Path Config" + + # Define paths for PS4 ROMs + gameDirOpt='Paths\\gamedirs\\0\\path=' + newGameDirOpt='Paths\\gamedirs\\0\\path='"${romsPath}/ps4" + + # Update the configuration file + sed -i "/${gameDirOpt}/c\\${newGameDirOpt}" "$ShadPS4_configFile" + + # https://github.com/shadps4-emu/shadPS4/blob/3f1061de5613c0c4a74d6394a6493491280bc03f/src/common/path_util.h + mkdir -p "${userDir}/screenshots/" + mkdir -p "${userDir}/shader/" + mkdir -p "${userDir}/savedata/" + mkdir -p "${userDir}/data/" + mkdir -p "${userDir}/temp/" + mkdir -p "${userDir}/sys_modules/" + mkdir -p "${userDir}/download/" + mkdir -p "${userDir}/captures/" + mkdir -p "${userDir}/cheats/" + mkdir -p "${userDir}/patches/" + mkdir -p "${userDir}/game_data/" + + # https://github.com/shadps4-emu/shadPS4/blob/main/documents/Debugging/Debugging.md#quick-analysis + mkdir -p "${userDir}/log/" + + mkdir -p "${inputConfigDir}" + + echo "ShadPS4 Path Config Completed" +} + +# Reusable Function to read value from the config.toml file +read_config_toml() { + local key="$1" + local configFile="$2" + echo "Reading arguments - key '$key' from config file: '$configFile'..." + + local value + value=$(jq -r "$key" "$configFile") + + echo "Extracted value: $value" + echo "$value" +} + +ShadPS4_setLanguage(){ + setMSG "Setting ShadPS4 Language" + local language=$(locale | grep LANG | cut -d= -f2 | cut -d_ -f1) + + echo "Checking if the config file at path: '$ShadPS4_configFile'" + if [[ -f "${ShadPS4_configFile}" ]]; then + echo "Config file found: ${ShadPS4_configFile}" + + emulatorLanguage=$(read_config_toml '.GUI.emulatorLanguage' "$ShadPS4_configFile") + + echo "Checking if language key exists in current language setting..." + if [[ -n ${ShadPS4_languages[$emulatorLanguage]+_} ]]; then + echo "Language key found in current language settings!" + + # Save the updated language settings back to the config file + echo "Updating system language and system region in the config file..." + tmp=$(jq --arg lang "${ShadPS4_languages[$emulatorLanguage]}" --arg region "${ShadPS4_regions[$emulatorLanguage]}" \ + '.system_language = $lang | .system_region = $region' \ + "${ShadPS4_configFile}") + echo "$tmp" > "${ShadPS4_configFile}" + echo "Config file updated successfully." + else + echo "Language key '${emulatorLanguage}' not found in current language settings. No updates made." + fi + else + echo "Configuration file not found: ${ShadPS4_configFile}" + fi + + echo "ShadPS4 language '${emulatorLanguage}' configuration completed." +} + +# Setup Saves +ShadPS4_setupSaves(){ + echo "Begin ShadPS4 save link" + + # Create symbolic links + linkToSaveFolder ShadPS4 saves "${userDir}/savedata" + linkToSaveFolder ShadPS4 saveMeta "${userDir}/saveMeta" + linkToSaveFolder ShadPS4 system "${sysDir}" + linkToSaveFolder ShadPS4 system_saves "${sysDir}/save" + + echo "ShadPS4 save link completed" +} + + +#SetupStorage +ShadPS4_setupStorage(){ + echo "Begin ShadPS4 storage config" + + local origPath="$HOME/.config/" +# mkdir -p "${storagePath}/shadps4/patchesAndDlc" + rsync -av "${origPath}/shadps4/games/" "${storagePath}/shadps4/games/" && rm -rf "${origPath}ShadPS4/games" + unlink "${origPath}/shadps4/games" + ln -ns "${storagePath}/shadps4/games/" "${origPath}/shadps4/games" +} + +#WipeSettings +ShadPS4_wipe(){ + echo "Begin ShadPS4 delete config directories" + rm -rf "$HOME/.config/shadps4" +} + +#Uninstall +ShadPS4_uninstall(){ + echo "Begin ShadPS4 uninstall" + uninstallGeneric $ShadPS4_emuName $ShadPS4_emuPath "" "emulator" +} + +# Migrate flatpak to appimage?? +ShadPS4_migrate(){ + echo "Begin ShadPS4 Migration" + + # Migration + if [ "$(ShadPS4_IsMigrated)" != "true" ]; then + #ShadPS4 flatpak to appimage + #From -- > to + migrationTable=() + migrationTable+=("$HOME/.var/app/net.shadps4.ShadPS4/config/shadps4" "$HOME/.config/shadps4") + + migrateAndLinkConfig "$ShadPS4_emuName" "$migrationTable" + fi + + echo "true" +} + +ShadPS4_IsMigrated(){ + if [ -f "$migrationFlag" ]; then + echo "true" + else + echo "false" + fi +} + +#setABXYstyle +ShadPS4_setABXYstyle(){ + sed -i 's/"button_x": "Y",/"button_x": "X",/' $ShadPS4_configFile + sed -i 's/"button_b": "A",/"button_b": "B",/' $ShadPS4_configFile + sed -i 's/"button_y": "X",/"button_y": "Y",/' $ShadPS4_configFile + sed -i 's/"button_a": "B"/"button_a": "A"/' $ShadPS4_configFile + +} +ShadPS4_setBAYXstyle(){ + sed -i 's/"button_x": "X",/"button_x": "Y",/' $ShadPS4_configFile + sed -i 's/"button_b": "B",/"button_b": "A",/' $ShadPS4_configFile + sed -i 's/"button_y": "Y",/"button_y": "X",/' $ShadPS4_configFile + sed -i 's/"button_a": "A"/"button_a": "B"/' $ShadPS4_configFile +} + +#WideScreenOn +ShadPS4_wideScreenOn(){ +echo "NYI" +} + +#WideScreenOff +ShadPS4_wideScreenOff(){ +echo "NYI" +} + +#BezelOn +ShadPS4_bezelOn(){ +echo "NYI" +} + +#BezelOff +ShadPS4_bezelOff(){ +echo "NYI" +} + +#finalExec - Extra stuff +ShadPS4_finalize(){ + echo "Begin ShadPS4 finalize" +} + +ShadPS4_IsInstalled(){ + if [ -e "$ShadPS4_emuPath/shadps4" ]; then + echo "true" + else + echo "false" + fi +} + +ShadPS4_resetConfig(){ + ShadPS4_init &>/dev/null && echo "true" || echo "false" +} + +ShadPS4_setResolution(){ + + case $ShadPS4Resolution in + "720P") multiplier=1; docked="false";; + "1080P") multiplier=1; docked="true";; + "1440P") multiplier=2; docked="false";; + "4K") multiplier=2; docked="true";; + *) echo "Error"; return 1;; + esac + + jq --arg docked "$docked" --arg multiplier "$multiplier" \ + '.docked_mode = $docked | .res_scale = $multiplier' "$ShadPS4_configFile" > tmp.json + + mv tmp.json "$ShadPS4_configFile" + +} + +ShadPS4_flushEmulatorLauncher(){ + flushEmulatorLaunchers "ShadPS4" +} \ No newline at end of file diff --git a/functions/ToolScripts/emuDeckMigration.sh b/functions/ToolScripts/emuDeckMigration.sh index eb2a95445..8bd5e2e8a 100644 --- a/functions/ToolScripts/emuDeckMigration.sh +++ b/functions/ToolScripts/emuDeckMigration.sh @@ -88,6 +88,8 @@ Migration_updatePaths(){ sed -i "s|${origin}|${destination}|g" "$HOME/.var/app/${RPCS3_emuPath}/config/rpcs3/vfs.yml" #Ryujinx sed -i "s|${origin}|${destination}|g" "$HOME/.config/Ryujinx/Config.json" + #ShadPS4 + sed -i "s|${origin}|${destination}|g" "$HOME/.config/shadps4/config.toml" #ScummVM sed -i "s|${origin}|${destination}|g" "$ScummVM_configFile" #Vita3K diff --git a/functions/all.sh b/functions/all.sh index c440e3f8c..04dd0b051 100644 --- a/functions/all.sh +++ b/functions/all.sh @@ -85,6 +85,7 @@ source "$EMUDECKGIT"/functions/EmuScripts/emuDeckDolphin.sh source "$EMUDECKGIT"/functions/EmuScripts/emuDeckPrimehack.sh source "$EMUDECKGIT"/functions/EmuScripts/emuDeckRetroArch.sh source "$EMUDECKGIT"/functions/EmuScripts/emuDeckRyujinx.sh +source "$EMUDECKGIT"/functions/EmuScripts/emuDeckShadPS4.sh source "$EMUDECKGIT"/functions/EmuScripts/emuDeckPPSSPP.sh source "$EMUDECKGIT"/functions/EmuScripts/emuDeckDuckStation.sh source "$EMUDECKGIT"/functions/EmuScripts/emuDeckXemu.sh diff --git a/functions/helperFunctions.sh b/functions/helperFunctions.sh index 329a95428..32cd4fd84 100644 --- a/functions/helperFunctions.sh +++ b/functions/helperFunctions.sh @@ -281,6 +281,7 @@ function createUpdateSettingsFile(){ defaultSettingsList+=("doSetupCemu=true") defaultSettingsList+=("doSetupXenia=false") defaultSettingsList+=("doSetupRyujinx=true") + defaultSettingsList+=("doSetupShadPS4=false") defaultSettingsList+=("doSetupMAME=true") defaultSettingsList+=("doSetupPrimehack=true") defaultSettingsList+=("doSetupPPSSPP=true") diff --git a/functions/jsonToBashVars.sh b/functions/jsonToBashVars.sh index 8d3235741..dbb127d82 100644 --- a/functions/jsonToBashVars.sh +++ b/functions/jsonToBashVars.sh @@ -47,6 +47,7 @@ function jsonToBashVars(){ setSetting doSetupCemu "$(jq .overwriteConfigEmus.cemu.status $json)" setSetting doSetupXenia "$(jq .overwriteConfigEmus.xenia.status $json)" setSetting doSetupRyujinx "$(jq .overwriteConfigEmus.ryujinx.status $json)" + setSetting doSetupShadPS4 "$(jq .overwriteConfigEmus.shadps4.status $json)" setSetting doSetupMAME "$(jq .overwriteConfigEmus.mame.status $json)" setSetting doSetupPrimeHack "$(jq .overwriteConfigEmus.primehack.status $json)" setSetting doSetupPPSSPP "$(jq .overwriteConfigEmus.ppsspp.status $json)" diff --git a/settings.sh b/settings.sh index 4eb9da0d8..8fc95cff6 100644 --- a/settings.sh +++ b/settings.sh @@ -14,6 +14,7 @@ doSetupDuck=true doSetupCemu=true doSetupXenia=false doSetupRyujinx=true +doSetupShadPS4=false doSetupMAME=true doSetupPrimehack=true doSetupPPSSPP=true diff --git a/setup.sh b/setup.sh index b981ee9e8..ac32bb408 100644 --- a/setup.sh +++ b/setup.sh @@ -360,6 +360,10 @@ if [ "$doSetupRyujinx" == "true" ]; then echo "Ryujinx_init" Ryujinx_init fi +if [ "$doSetupShadPS4" == "true" ]; then + echo "ShadPS4_init" + ShadPS4_init +fi if [ "$doSetupPPSSPP" == "true" ]; then echo "PPSSPP_init" PPSSPP_init diff --git a/uninstall.sh b/uninstall.sh index d509066da..99423ca25 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -515,6 +515,16 @@ fi rm -rf $HOME/Applications/publish &> /dev/null rm -rf $HOME/.local/share/applications/Ryujinx.desktop &> /dev/null fi + if [[ "$doUninstallShadPS4" == true ]]; then + # Flatpak + flatpak uninstall net.shadps4.ShadPS4 -y + rm -rf $HOME/.var/app/net.shadps4.ShadPS4 &> /dev/null + # AppImage + rm -rf "$HOME/.config/shadps4" &> /dev/null + rm -rf "$HOME/.cache/shadps4" &> /dev/null + rm -rf $HOME/.local/share/applications/ShadPS4.desktop &> /dev/null + rm -rf $HOME/Applications/shadps4.AppImage &> /dev/null + fi if [[ "$doUninstallScummVM" == true ]]; then flatpak uninstall org.scummvm.ScummVM -y rm -rf $HOME/.var/app/org.scummvm.ScummVM &> /dev/null diff --git a/versions.json b/versions.json index 47b2675ea..b0f563f31 100644 --- a/versions.json +++ b/versions.json @@ -11,6 +11,7 @@ "rpcs3": { "id": "rpcs3", "code": "RPCS3", "version": 0 }, "yuzu": { "id": "yuzu", "code": "Yuzu", "version": 1 }, "ryujinx": { "id": "ryujinx", "code": "Ryujinx", "version": 4 }, + "shadps4": { "id": "shadps4", "code": "ShadPS4", "version": 0 }, "xemu": { "id": "xemu", "code": "Xemu", "version": 0 }, "cemu": { "id": "cemu", "code": "Cemu", "version": 0 }, "srm": { "id": "srm", "code": "SRM", "version": 7 }, From 516288d5515e855e39e0be89a9fd5ba4cb5448a2 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 23:14:34 +0100 Subject: [PATCH 182/230] fixed missing artwork --- tools/retro-library/missing_artwork.py | 47 +++++++++++++++----------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index 78b503228..a792bfeb9 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -2,8 +2,8 @@ import json import sys import re -import hashlib import subprocess +import hashlib # Define the log file path home_dir = os.environ.get("HOME") @@ -45,10 +45,8 @@ def getSettings(): settings = getSettings() storage_path = os.path.expandvars(settings["storagePath"]) - # Function to write messages to the log file def log_message(message): - print(message) with open(msg_file, "w") as log_file: # "a" to append messages without overwriting log_file.write(message + "\n") @@ -60,10 +58,11 @@ def calculate_hash(file_path): with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(65536), b""): # 64 KB chunks hash_md5.update(chunk) - print(hash_md5.hexdigest()) + print(file_path , hash_md5.hexdigest()) return hash_md5.hexdigest() except Exception: - return None + return None + def collect_game_data(system_dir, extensions): game_data = [] for root, _, files in os.walk(system_dir): @@ -76,11 +75,10 @@ def collect_game_data(system_dir, extensions): extension = filename.split('.')[-1] name = '.'.join(filename.split('.')[:-1]) if extension in extensions: - # Custom logic for specific systems + # Special cases for WiiU and PS3 if "wiiu" in system_dir: parts = root.split(os.sep) name = parts[-2] if len(parts) >= 2 else name - if "ps3" in system_dir: parts = root.split(os.sep) name = parts[-3] if len(parts) >= 3 else name @@ -93,6 +91,7 @@ def collect_game_data(system_dir, extensions): name_cleaned = name_cleaned.strip().replace(' ', '_').replace('-', '_') name_cleaned = re.sub(r'_+', '_', name_cleaned) name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') + name_cleaned_pegasus = name.replace(',_', ',') name_cleaned = name_cleaned.lower() rom_hash = calculate_hash(file_path) @@ -100,6 +99,8 @@ def collect_game_data(system_dir, extensions): for img_type, ext in [("box2dfront", ".jpg"), ("wheel", ".png"), ("screenshot", ".jpg")]: img_path = os.path.join(images_path, f"{platform}/media/{img_type}/{name_cleaned}{ext}") if not os.path.exists(img_path): + log_message(f"Missing image: {img_path}") + game_info = { "name": name_cleaned, "platform": platform, @@ -108,16 +109,16 @@ def collect_game_data(system_dir, extensions): } game_data.append(game_info) - game_data_sorted = sorted(game_data, key=lambda x: x['name']) - return game_data_sorted + return sorted(game_data, key=lambda x: x['name']) roms_dir = roms_path valid_system_dirs = [] - # Validate system directories for system_dir in os.listdir(roms_dir): if system_dir == "xbox360": system_dir = "xbox360/roms" + if system_dir == "model2": + system_dir = "model2/roms" full_path = os.path.join(roms_dir, system_dir) if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) @@ -127,7 +128,6 @@ def collect_game_data(system_dir, extensions): game_list = [] - # Process each system directory for system_dir in valid_system_dirs: if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): log_message(f"MA: Skipping directory: {system_dir}") @@ -135,25 +135,34 @@ def collect_game_data(system_dir, extensions): with open(os.path.join(system_dir, 'metadata.txt')) as f: metadata = f.read() + collection = next((line.split(':')[1].strip() for line in metadata.splitlines() if line.startswith('collection:')), '') + shortname = next((line.split(':')[1].strip() for line in metadata.splitlines() if line.startswith('shortname:')), '') + launcher = next((line.split(':', 1)[1].strip() for line in metadata.splitlines() if line.startswith('launch:')), '').replace('"', '\\"') extensions = next((line.split(':')[1].strip().replace(',', ' ') for line in metadata.splitlines() if line.startswith('extensions:')), '').split() games = collect_game_data(system_dir, extensions) if games: - game_list.extend(games) - log_message(f"MA: Found {len(games)} missing artwork from {system_dir}") + system_info = { + "title": collection, + "id": shortname, + "launcher": launcher, + "games": games + } + game_list.append(system_info) + log_message(f"MA: Detected {len(games)} games from {system_dir}") + + json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) - # Save the JSON output - json_output = json.dumps(game_list, indent=4) - home_directory = os.path.expanduser("~") output_file = os.path.join(storage_path, "retrolibrary/cache/missing_artwork.json") + os.makedirs(os.path.dirname(output_file), exist_ok=True) with open(output_file, 'w') as f: f.write(json_output) + #print(json_output) -# Read ROMs and images paths from command-line arguments roms_path = sys.argv[1] images_path = sys.argv[2] -log_message("MA: Missing artwork list generation in process...") +log_message("MA: Missing artwork list generation in progress...") generate_game_lists(roms_path, images_path) -log_message("MA: Missing artwork list process completed.") +log_message("MA: Missing artwork list process completed.") \ No newline at end of file From 245502a3fe1e14f7f29d32be975da2ea8596dcfc Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 23:30:17 +0100 Subject: [PATCH 183/230] shadps4 launcher --- functions/EmuScripts/emuDeckShadPS4.sh | 43 +++++++++++++------------ tools/launchers/shadps4.sh | 44 ++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 21 deletions(-) create mode 100644 tools/launchers/shadps4.sh diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index 26d340184..18986dc49 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -1,5 +1,6 @@ #!/bin/bash +# Credits: https://github.com/Aeonitis # Script to install, initialize and configure ShadPS4 on EmuDeck # Note: No Bios/Keys symlinks necessary @@ -18,10 +19,10 @@ ShadPS4_emuName="ShadPS4" ShadPS4_emuType="$emuDeckEmuTypeBinary" ShadPS4_emuPath="$HOME/Applications/publish" ShadPS4_configFile="$HOME/.config/shadps4/config.toml" -userDir="$HOME/.config/shadps4/user" -sysDir="$HOME/.config/shadps4/system" -inputConfigDir="$HOME/.config/shadps4/inputConfig" -controllerFile="${inputConfigDir}/default.ini" +ShadPS4_userDir="$HOME/.config/shadps4/user" +ShadPS4_sysDir="$HOME/.config/shadps4/system" +ShadPS4_inputConfigDir="$HOME/.config/shadps4/inputConfig" +ShadPS4_controllerFile="${ShadPS4_inputConfigDir}/default.ini" migrationFlag="$HOME/.config/EmuDeck/.${ShadPS4_emuName}MigrationCompleted" @@ -172,22 +173,22 @@ ShadPS4_setEmulationFolder(){ sed -i "/${gameDirOpt}/c\\${newGameDirOpt}" "$ShadPS4_configFile" # https://github.com/shadps4-emu/shadPS4/blob/3f1061de5613c0c4a74d6394a6493491280bc03f/src/common/path_util.h - mkdir -p "${userDir}/screenshots/" - mkdir -p "${userDir}/shader/" - mkdir -p "${userDir}/savedata/" - mkdir -p "${userDir}/data/" - mkdir -p "${userDir}/temp/" - mkdir -p "${userDir}/sys_modules/" - mkdir -p "${userDir}/download/" - mkdir -p "${userDir}/captures/" - mkdir -p "${userDir}/cheats/" - mkdir -p "${userDir}/patches/" - mkdir -p "${userDir}/game_data/" + mkdir -p "${ShadPS4_userDir}/screenshots/" + mkdir -p "${ShadPS4_userDir}/shader/" + mkdir -p "${ShadPS4_userDir}/savedata/" + mkdir -p "${ShadPS4_userDir}/data/" + mkdir -p "${ShadPS4_userDir}/temp/" + mkdir -p "${ShadPS4_userDir}/sys_modules/" + mkdir -p "${ShadPS4_userDir}/download/" + mkdir -p "${ShadPS4_userDir}/captures/" + mkdir -p "${ShadPS4_userDir}/cheats/" + mkdir -p "${ShadPS4_userDir}/patches/" + mkdir -p "${ShadPS4_userDir}/game_data/" # https://github.com/shadps4-emu/shadPS4/blob/main/documents/Debugging/Debugging.md#quick-analysis - mkdir -p "${userDir}/log/" + mkdir -p "${ShadPS4_userDir}/log/" - mkdir -p "${inputConfigDir}" + mkdir -p "${ShadPS4_inputConfigDir}" echo "ShadPS4 Path Config Completed" } @@ -241,10 +242,10 @@ ShadPS4_setupSaves(){ echo "Begin ShadPS4 save link" # Create symbolic links - linkToSaveFolder ShadPS4 saves "${userDir}/savedata" - linkToSaveFolder ShadPS4 saveMeta "${userDir}/saveMeta" - linkToSaveFolder ShadPS4 system "${sysDir}" - linkToSaveFolder ShadPS4 system_saves "${sysDir}/save" + linkToSaveFolder ShadPS4 saves "${ShadPS4_userDir}/savedata" + linkToSaveFolder ShadPS4 saveMeta "${ShadPS4_userDir}/saveMeta" + linkToSaveFolder ShadPS4 system "${ShadPS4_sysDir}" + linkToSaveFolder ShadPS4 system_saves "${ShadPS4_sysDir}/save" echo "ShadPS4 save link completed" } diff --git a/tools/launchers/shadps4.sh b/tools/launchers/shadps4.sh new file mode 100644 index 000000000..ead2d7c3c --- /dev/null +++ b/tools/launchers/shadps4.sh @@ -0,0 +1,44 @@ +#!/bin/bash +source $HOME/.config/EmuDeck/backend/functions/all.sh +emulatorInit "shadps4" +emuName="ShadPS4" #parameterize me +emufolder="$HOME/Applications/publish" # has to be here for ES-DE to find it + +#initialize execute array +exe=() + +#find full path to emu executable +exe_path=$(find "$emufolder" -iname "${emuName}.sh" | sort -n | cut -d' ' -f 2- | tail -n 1 2>/dev/null) + +#if appimage doesn't exist fall back to flatpak. +if [[ -z "$exe_path" ]]; then + #flatpak + flatpakApp=$(flatpak list --app --columns=application | grep "$emuName") + #fill execute array + exe=("flatpak" "run" "$flatpakApp") +else + #make sure that file is executable + chmod +x "$exe_path" + #fill execute array + exe=("$exe_path") +fi + +#run the executable with the params. +launch_args=() +for rom in "${@}"; do + # Parsers previously had single quotes ("'/path/to/rom'" ), this allows those shortcuts to continue working. + removedLegacySingleQuotes=$(echo "$rom" | sed "s/^'//; s/'$//") + launch_args+=("$removedLegacySingleQuotes") +done + +echo "Launching: ${exe[*]} ${launch_args[*]}" + +if [[ -z "${*}" ]]; then + echo "ROM not found. Launching $emuName directly" + "${exe[@]}" +else + echo "ROM found, launching game" + "${exe[@]}" "${launch_args[@]}" +fi + +rm -rf "$savesPath/.gaming" From a3d42c1c4047fc2902226111dfb3fcb4d84f8088 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 23:31:38 +0100 Subject: [PATCH 184/230] ShadPS4 icon --- icons/ShadPS4.png | Bin 0 -> 133377 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 icons/ShadPS4.png diff --git a/icons/ShadPS4.png b/icons/ShadPS4.png new file mode 100644 index 0000000000000000000000000000000000000000..7ad301d4c66869ba0dd40378ca461ab5bc5f67d8 GIT binary patch literal 133377 zcmXt;Wmr{Tu*UZ}aOm!k?nW8}4kaMnAYIZW-5eUELsAJ5kOrl@L%KVpyE`xcd!PGh zua9eH&zgAW7pbBogN{Oi0ssK|2U#gK006%p!2lA%>*vyQ>K*_nfe%vR8lHxSEr`vZ zK5EjpI*f4i{WX|ISA<*ssr6G;N_vVI5055?apP890ZE7kOf8gS+TL!Y0t&t3{t@(r zPn-nXN+QWRC(uDiL)%wy>D#F9>Wby6&vC=QrJG|P&5GhtYx0xn7KXz=*DJPuEQsAu z=^wC9AH`SJo-4|-nEAf^3G}YiDo$mXnVG?~cXafdyuQ9pdLJ}fpgv$lpM)F}6Z1is zJXJ%8t;&-&{)04aR)idNg31O`SP%{h!l@o+$=+L`t5ZKw)+K1==;*2AiF$is`?PYP z^m3wF18t9l>ffzKJ4;)w28~qHR^zYCTe^%y`hU02hn;AhjQ*_XlVr*+zkU1mEJlZw zE=Kyom4~E;{48P~apZea|9D;?&aESW_%S))FbFJav`RHzxkkM+@*Q=@d4`E~NQhvo zE~O;@AgX7ArsV5+4c4*CfE1S^6=oH2gfxv7eG*c)DfJZsbkEl~;-xokEB+D`t^q{? zjKgWqm2a5VUau;qj1oUub4%Lo9{ht88pgqwY5MgGjulU)R-S6AP0f$wA9DGIEcmD~ z-jfSb>B1z1}j zc-D}3dh=xlSkU4$dzm`hkuP<3zUfY;b=77Qc!R8J^W+!lU8&p(#>y>rvcO(FL1W?%yj+vD!3t3 zxotU6+vTfrTd-o=$aoD1mO3%{;KLuV^-(-O)LIo`vg8v8OM_XlM9mStw(g?ls^$uD zaBJVkr0QwMw!)wu&N#<;TJ{v%Xp;&PAnLBG)$&0Ly7I3ztnj~CdOXafj5X&nf8Y7z zhdkr0o@+sa(bJzgs(>B9IPK^_4=A@o@1He(x;uCe;y5)eZ@B$krW`GT!8J;=4mb{OMXy3+>M>05GqflhKb@+1yor2 z7`%)bEIemE;GHr#jWKb~b?$9DI6)#JFI=3ZzZHrB{v;}+hfuv!WOjH7$hn25X_;;r zsIGKcIKcl-c;xPn2J^LtOx%dRFJmv7@K2yU6jkf@{L*8r^V=J0_-J8kQ&TST7#Rrr zgBX#u2XDwvt|ttl-)5@cIS&w&Q8uAA-Qa1~|Y zWKP~#gY$+C07hUanN#T<&D0-8BBfQuIMWkFcoHRL%0Wtz?Y+9BPua%LBPif8PxldX z1XV!hcWbL?ZH;CC#v_y?Gb>9pSuPYzC3TLw$-9fv>8M#D(|27WkBJBD){M0P?MRs$ z#v?zV%-LX3NLX%rKK^D`v{i&l*gwk_hd=Tkwje-8EvC-24;+>_m}nmZ$p+hM7C1oN z{5`$Buy9SNQvO6I6IbBamJtN;_%K67e~**Z`Rn<$NS+2OmfRQTg<~2K#)Jk_X(Tx# zZd%h?YpscbfIgphgxT!=<9@Emy@&VSY0F$gf;39f&=65Od-T^7jk`AKSRZ<)Vg!{; zc_4{Ri;n&C9igkdt#y<_qUpqiE$Sv78rF%|tkQRwBXV^j@4Q>Ze;Dz_y(PsV9A!(k z?2AC4eoMpeW_KUv1n>8(U++3QC32iTgf+%tHfT((q-q=2w7TrfVB!x>Yv-$Tt2TB@!OQ?3NTiTS@oKaXh6hVu{=NSKzdkO z??9|mnkm_`oGDuvwA`U*=<8G8H_!=54azPcGjRLcviYtqRxhWKf?p1cG8N;mHj8EO z$Jqp3Lx2(}mxXQ)<&QIN$mB84%jeX8cJ4`L9M;o;hV~vbG590VB9E4!j>fxSIZI4jK z^XAG?FG;(?Dx|9A*WFV(YEVil^}zxDZK7P_vUJOSFr4t%3~y_CP+k6c;53=QA;P%{UJm9JyPC+m1=VGEinaddNzVAkeJ((Q-fp0Tf+w5 zTWq6~bKGf4aa?t?mSotZygJZ~Xh}o;RrkGVLVZSH*|4*jt?fmu742gXjf}-f%fvd| zwWjM~jZV-E^dqywGrIx@hmQU?Oqy(H4hr;LY;DD|iL%;QyH^f#@&(Cqql{lrN~=y9 zF{OoPC=pSy?)^W+0GcT=o4H744?KRx*;8~;qh+jy8}bB1a@N6XcLOc0i{FjE*}mGS z8+$~BzY4jjao&NQB(mdhJJ82B2Kp>Qzl-7M zkw3Y*;np?gu#d&7`Rnw~RZMPHshz*jIC&d`&O2^iTxHIdr3Y= z@>$_>Hp{w5Fi_@PRio;lk#Vf~=RQ-XjQ(uL_$9{3pu_o6aMhVug$8t(%k}nBb8&x- zPnxCOY&?Lc1f>Q($^6@)jLNT}{J{)aP{KJqa6d8OGUT))|I54t!oPet(IDBkEyb}x zvU9N4Vwt`lWS}<<5aH#mY3b7RomX1jV=+nzoR|&R9=i9ivvhG3aW{V+yY-}2d9NYO z^JQPDT(-?0Y>x%3Nps;BxEexP0sY|HKPWzWJQ%B_GZ@@R%xSJDkB(w%t55C4zFW2| zzgwrpTC9n}sLj;MLXB-Cql`UQRX*js=dg^i*-2}u_TZ?4&0cjW&27Pf^K3l3TL1Zf z--!!oV|=&dSK)e(=|6}i@PUj(jBP4;Nk`0?fjcoqz1^D1+AizqxYLj9{hFB2YBJC7 zdCAP(pW0w$TWJ8xm6F^mS{>`x+)v7bp8~O>Kzp9TUng~ewNjW(cV}H`r<-nyRppo1 z06`sWFqa^4`~k&Vx;Xz$?1p1q9n^wq)3A#BZlP-Nq)Z3Hl3%M>#6z+T?KoYbrqc^6 zL;fF1`{iT5rQMG8a|9qeIIV%&d!ia-9PI2~@DwCD^!2|f<%q|{X4yibTo-;{NF z^Nxs+SK@Z=Q&!apAZC4_*bOrNmRt)@eXID?;*dM2{1e46s`j8Vo^_;fE9_u<>uubZ z!8`}pI6*22XZt-zOYAbrbGc7on4J(jr*RppBY|Yc?n@dp`TikzCG1uV4z{)RsFE)wi&y1IGFC-??TK)80TTrn&`bYg+={X{UW?{CS>7=iRK__m!ZZoMhAAt59`)jWeo`kav3 z?!PQ{w>?m(GG#|o3=(svXN5eLfnWrOB5~vZ>@ngdm|wiJUKsK*U0>S0T??BSoB~F` zUXcCDWqy261-&5#h#>u^AEfDt(np!gsov+gkL7%7^I)Ms%YR%q(ZzC3jNaayONjS9 zxe-<)ztMqFPGE8Y)24bd;SzgJyeC$%-zk9`=*8Ls3G<}L&E`PlqPuIhywiCg7$~ts z39tRzFsq~#{j|HO0sii@fM=!HNT~V^Xs042nwOJILngQCtcauZyIUipK6!|H*m~?? z1;jO&-QZ6l8>g!@)p&4L#is!av6o2oyUDyA(tY-zk%xmAYrb+^=gY-kqOaEnJlVqJ zPg^*|I6w)A!s~K{FEDKvFpSsgDMr$IU=e6Pr5PoPAn2o+{EfbXmvL(e{}Kzs4EyaC zX`h@dPx(>28D>&TR(!XF#Fbu*-Y+E&1@1T?PC5U`Mmg?RfD`^sr9QIx0SiX0;lIgQ#Kef6L5di&q6(KFQuP^Qv?vG57Qj{>FMlm?SGmy{|nGE!^ ziGqZ|19G?awl_QGBZcj|KO2)|*;rmT4^WOB{xvX?qM6C~ZO>2K8{9XiW1n4OIFDQ< z-i5w@NCv>Vs5yQd6u@CnB!%lsn^(TS#A|}d2O(+;Jw2!lShaoqMTY?3738UX8SA!3 zCFcOlSDPsEFhjd=Lg^@5k;NM{6X)C&69Gsy0wd5VY>h{U67Rsy;akvdQlO?UU*A6&1E;G7pUa zLU9z*-c%>nBCQf|tLbt%D{LYC&i50g+NB;3E zh3EG0xxp%#DqV7BB>@9bco?C%=EtP>1^Yd7Vx6!c!;04iqMSCoE>J$F^d31`ln?_k zo134o%>!|B;X<8*1(@rfaj;I_Kgmdo7#&~7q3Y42AvwHqY$&ozWIPcS{fQio+jTjP zqJGEzrFoC-=5GV)r?0)*js;k^sS@H8zQ}lmkY8` zz9Nyn!nF_imCOJyxbX%I7<6cm)_JfzPG6Nr_;@7nsLOv!42XjNvop8+Q!uGn<(M!qu%t*Wt2A@P&jX7(D`AL zqgj)W<9tN#&-rn{i_kZrC?`nHxh9e>RPBzre-x=G{P!m(27suV(uYk)yQMS`wQ=?; zPpKFakj`KGHEYxY{xy6hoj>kx7;rVQiVU1{lK`9JB87iZ>e$q8cL6cv03_3AjCt_9 z1o{OJ3WRQifHtrmMpO!ZO8}Kz#wBx~#bA5`0WS53kJqd&9HngpG5KG(H~4qz*?@f) zL}QyD!cJ(Pu?}C<%6`b1qV;-@)y?#GfGe3teHYx${Z_vw6n!wjPx)v~t7@q=PYH~D zkyZ&R0u8z%IU}BJ?MDsAECeUWZh8SAiq%nriIbI72*6rl{~|zplC7i|n^%j!bc*ot zC=py>$C1bO@#v1beX>}hWyQ*snJG|f76B4SNuHnu17^QabIctVN0&1HvNvA-!MQvux|nxGjt1Fx zyZpk16-aa(eKC(`!WU1X`bvWCUy^ruvolxZHa#dZMxIaprxCga8H~1KdEcBUE5*=^`*&-_;oXXT8rmo|VMqDpd{t1l%t6t#Ax!-c_aQ$a zjH77zwll3n7oG9c_h^(05eOQj-6}mUtX(D#2q#lis!REoq^=2rX@7!n7z{wrjF)z;!7d!PWV7Q8QSk0-#>f?WPp)1$*mgw-Sk zJItH+FUHYMo@0ZjePn|kDlGH&T-uverdI)u0ly?e(()N&7)+@qz((LGuwXfsr60pGtx`;Xi99B$pbTFqLvoeg)=XEZdT z6g?i*eXrDb7ke?+;rs)NQXM`hr^Wx?|HREGIX@;MnS^l`HQ9zL=xdQ5$^3kz&zI*T zWNvUPzByMrLk~++u~fsk@jPz3mnSHq<=YU;g<&hu*nF`)`I7TrtUEI#s!2yJah*?f zEa(F5QiC3C!Ti9O8v;+8riovz_2QB|vY2!S%|!OG^EFO{BR_3en8e0BjCPJP2BH~e zO|jGme)}su6Xl)Lh4$pjEcwq1pHH&azt(q#P@Eb?bw&N?C=S66m0)-09Q|4^F+hk`?Fw>QLS{EJH*Y>$+W|IyMU}4F3 zM1|i!Ax5LgB?6S-mo|rKTm6hp+m@)v7zDu8M}GxQRcWkqz*6#5gcchstzS1$EdA0P zyCdi{b5b+sj(MrXJ%6g4f0+uf($uZFE4iy%>^ng~IzCz6-~@7}-;6*QqkSnke+Rjl z$9Hr`=LKdVBma6DkQMJu)H{F7_Wb6{tq-=6Es;a87GGG-9Dj;r`efG|it~QoCVtd3 zCPSj?dRmt8yxLW$Qh2?WeE(b?46qBQ`EkLYH(6M!|G|2Wezia8_61H;|x+F(uH zSR0R$!t6NUER1PhR`r4ZOZ15TgzpF{yccuRx-QBl4ozrkLLRcUw{%Lg_n`S}|>z3WVLlJbYV4$_+~X zk1}S=>|zX+xrsSJU=o47F1;eRR;mn?j|T^#6+LD}mI?~+q3k0a z!&W$fp&Q@41SmruIEFBAvnn2HN+*? zu96OylrxFLk;Wb?!=RNu(ijch`;Caot{DaGCt-PHpO#gtN1wTSa2U0TM#0ttO|YVA zLg^d5r4+)KbF(c0eK4N=8z3 z7om3ljN-^Mn~2NP#Y*;q;A(1-$9$;utP&eib0o=;)><#9EAE1oWm~x*lNt!O0NDcx zk~CH4T*4_oZgl|s^~vGu?h~dQxU2%}jO-X@eOCAG{%J>Jx3Kwj5>lB8BH~R!5ZJ=1=l`aV7*Nx1`b^f87+6|V?$K=vW^4Aw0d#%&PiV` zEWB=!HLz%rDvdst)CGg&UW;|&l@|8#S6|VW+|awTm#`fwG?#!_KR7^MUaL4>?}Y{a z@kz%*KdJlY!JhVbg(x0iTuH$X=fCQe+E3B)?0$QXxv`3LZPo_-Kz&S>qx(Rw>D+-4 zJC3!9LB5&2|9__BGPN5+!GO*4*8>JR#2JgCbzTfm6XN!ZINUi77BfE9QAl4-p>$m_ z&XUx=KE7fY5qEpCt-5x5?Dh@Hk7M$|XI&sH3+ zDF0)o^AMQ>LomJCkbL7M7?9Ii57_!0+TOlo#Lgc3B)q%*KH_`p7ok;Ke-AhHf`{dY zt9?QffL+0~fRFqb<%@Fr`qDUYAV>f+nnhOR;LLd1re-!?q2v7BNdo!w z&H6J~A}&}91?3*A48)o#Z`Z#jqI~fL(#!NWBq|QK+Wqg|SYqdr{p>l|NyPl7>9M@!?+`Iu)ZdU_d{<^G> z&t(!4^yVGz&-xe(4LPh`F0=-CtFJs{Q#(Gr7Sy6cpn-nWg zUQDxN6>oWvp+pC7Gg!W3r#4Z=VFK=wye}mGJ3ys^S?~0uY6QWsPAYUjS7Z<4l_20t z0aM@R(otIEieZKL<1qyfqkc>m&on_mZDd(9AOm1jKGlE<0u@3`3e4gI*Ffpx+8=VZ z)dvymF*B#CCWkxA&2+7JRRN+GNa@8+omSp>zoLuFUVe+dn-2=$EZ@)&<9z2Q6E`HB zGT`qWgMk2ujOL6|7*!r#lCCyWXWtIJXG&pkDtvp@@VD#|)NWV~_hR^;qu+3%U;&$y zmn=@8O-#C)Va5nS>{{jwO(rT;#v1&eg2zTFML26T z)bc~%@{5%yx;+ZaIoOac^CyMI+M+8`LpQisoB&}PQDYF&7>RmUlP(h4o7I#;k=N1g zMaF{81QGs{)Ov@s5d=I?h0Tepq_CF<2oPlY_vT6lu2){{xUFP?fV5}H)q2M*3Ji`B zzX0Jt*EWNdkN7t2LH+CatIs2)%{!7eRCA6fNQolEHKVIu)xoCChBd1hLxa}=;!Q$o72+lGyv6Tp%Rtb>g^V$A;`=nTktwyqVV(ELJR(aWq3_n3G8It^X(bBZRj?##XJzpK93{D4mtL0}K2f+ob4Y7I|u;VjRM zIZqfq&a@vo*PD9rA(Yw#+CLIjTK9bc#ebnsi>diz-^9--@Si3wh+0UqW)YT0AwTPe z*371MPHvt=!rD$wcAx>AkjJ~#)hFKl`-1=xP`g$m{l5(o0PmkN-`8VCRwmCD0DwdG zQn_#^*!`@FfM9dib!=4pq@d_(bl za!BFdNCBK_0pYZ9X)rghYi@`5lb4l{pRgDKBNW#3RycB;w4x9;Vo;)eY3g81*yHQ+ za-GeP#WC&V1|FbXeUVOces(c2v72Ce499jL1{}-pd<@kJkf%Ry$K^hSR zf8^ZZpz;q6sIb07aLu^OF;BUl5v2iB8!-fXo7B4$2X&G zBM&2^*-n3Qce>My{|t(0?#?SYZ{88nYCW+2{9znt?VEryFeb>~?Mr$vJ?ts$-DQ}J zN=^(c1-zX5md!kl=Jjn?#7V&ga&C7u&^8zJ%_0A~`}22|;TZ2OX*NLiZx0Pl>o9C9siK++y(xn4&`H4`521Oe>z zKe6Q=P#McWfZn=Mo+HLj>1$Q$8$P1`~-a@%0+Dnl3H*C#b%O(d)* z$pnZVpgn$KO)-O`lO5jG=LgI>xR%}e#S_2#lN4uoAo{t2jh-$X>lewll3tDDPtV%l zf*mvw!?iP@qAEu^l<~;){YIH^YG>|fz#DdeW{Bxnr(f}jG{f+XezTY$=&45=2bok`o`8+GR9j_)mmcnx^|nSgD?CB2v1WZ zy6u8yyWyQnr$5!-TciMuVdy5j9D1=dg zh11GOHAI|z^EABLYH*$}iF7a7-B!3K1MRH?JgZWJo5ioeYquxRH1P8y#ebpbD#-JF zfFkyT8x<1-fFUH8d6h25mJ~YZ=!ijn7?3w10IXM;qvgxqySqc`)cuc6sQ$}; z;yC(dZ3!Pgx3aQ9-?N5oxh1T64HKGs@j?{fhbgdM6q^n-S#(TOId=W#r$;C~?){H1 z{7^$~NJw0JmG+tX5XYh`uquSv?wUhS?s(RbcDQqoyvT(9pBv=FdKi%CmCuGSK zl2x}LW>eu9Sch)<)__rXlfXrAeNVTr+$je|P zF0Lyi?O5`iDPi=#B^i?q%o~&k%e4LtC7DHZJ zkkBnw!S_G~ZsD1qH(SPW6i)R%A!jYB4Nqps(Eog!5RV{-F$=ci?T!sTdMgMLli7w1 zD=#f0`MfT2!WRJ8=$LC{u85sR4MF(_tCo=LX4=LwZzDc6E&90wxbD@6)MQ(+pt80NCF^_(3y*x+tL)&&r)JpPc@xma$a>_Ru zC~mx9-ymeHtv#eQxJl|$-yi$^`X?*;?3A4a2VOVgQQI8B5R|Eo?e!B~#Wv-VBih9e zLdWurf<9`0rp*s_{~>&y_;@#k^&XWaO9{EN#-bEue6ce-bK-V`DF5J+LMRWW=vwIw z-9eyOi#^N876;zHrH|`}R@`0aUMekI3(X!pzcP7Dhm($+e$6v+4GRX9;jYUHd~`lf z^?3Bu6Cro2^|0^{Af6lEhfI1;zTL=N!vH61y@{QR#K7GF8Yrg#Kl@d~Tc*5SEOpH9 z?4X|2etEk<#pjJp%mT{Hf4a*MV||pcrype(1HosQ2w-}Z2Z7p;i3_1_1MST5#WJU4 z{&rcPJWG?-4fEq~GG8uJDO#lJ3*HN8Na8lpc^zYl8u}l8OvCM!iN`P;8hSRoX&uMw zKKwW+uh-yYwSM}FI6*hr7N{)>2~{4Te%==x+uMCFGF;ebDi0J>5q%0!dCF=dE=Dib zZjM!!^L^+&j_-5$Pu;TOV>!hz0U-R;$=P}C@(PTbzc z+^|T9-`6SQ0XrDNp)C}MED!4W#R}#@jInSpGP0jmV1@^?QJ@E~9Vf)^9=%}n*Ma5O zGCxiP$dgAMDB`Rcev!s%jP@3B%$Gx4OwqWX!G#CLo*7bD5-mFL8EF^Qk|qR5=0`dv zI0+I+;4sciH?i45I}re8*w~%1p5Q4MV0&H%10&)tzHiMDeyTF$eD;@2gyQs57ug~jdhyKBeQjkKlon+h!~RJFp^<%vI1j{zvx&%V)+yC z!&jXLm}DeA<4UOTM{{6Udz)jVbXT3C5?x~H)d|YI2l9!#MXDE82`Mg$z7DR}UWq8j zpdOPsra3K1j9l5sn}m_BcSxVA(f2=rtSAXO>_KClsx;a=9!s2>z8__(9pIpv@be*D z$R;0@Cl|Q+QV=p`=><{SzE-*ogV}@)9Kf^tYrM_1TA?L91anJbH4-1*Pfc)Pg`8p% zsLH~FXx?6`#d1-kHq@~;caR!ixYu*NIZm1BJwxif1i?7O5LMd|u?(?lo?8A$vERJV z17ceRg7Z@Ej9uyt;<6~)4~-7(k6#oZYcH1>L-s)o$@>vgox#*pMKBibgB*~Dq#rXo zU2-|gxMjP9$bKb09O9n|mxE%sHD=I(D^&(7yx|sk9SBL4vqucS%$;g#--?l^jx3o6 zuBU%e;7!wObLY`ZbD4L(1qWnUd`GO$S8LBu3(tRt&oN+bW8lAo&p(J}vz$M7Xl)f) zx)(e2PRBoZX4|4j7@GPG!N_uBDK{>cqL6<0cjC=z+hsyt9|5`YFqDMYS1`OkavSsM z&Gu@9LA>_kiO=To11v;h;}>E~>t9OQ76cOr3+$r)4FIF(xN6lmpM!59$f!cYl2Wg~ z(04yUDcCTj!@^I8{}CIWd=mSXjaP5?!tp|q(Vn2%%aVb}iCfuz`{bKm{+UaWNx$m1 zoCl|*IjmbZm5Ly?*dL^7r2x{}E1fe}BX3XtJhxMqQp~d5nP+6CZEbP$cku*$w`Rf* z#FHkGXH)r_W%dqgo;}?LgJf#qKyr0NUMOrp4e1RA(B0wL8}F>$E!w2&}pwxgy4C1D4W+tkJ$IsY|%g0oqI)*&=`jJBV~E4pQ`om9^2d^ zv2VR?zAE78(1|umZ<&$)hdpPPK>W+%34TgMx0DjhLV`U}^Ffjt*mZnY%7wbKlWiu~Ok%9ipQ z-L|}y)wUHxtYF5q8qshAX!M3EB9B9aS$=|JoMfQqX0;s%>GEB(K+8Y@HD*qnA4!?F z{EdEcx^NI^4h$^kA^ls%XJ!l}5DN2k8qWxfPL3p*0GXhzfcsyd1&!48@N;yr9idLQVb>R>xGvNlH=d`E2 zcx4HCTaUH)TD7cn*}9aVn?s9vF>i;6f12%DmJ$uY6;vvZ(<04^*>i_y2ZgLL%XTSA zMu{i49Z^a{00bCNad-CP#ow`A_2k3|>KT4?^C*h9>sb2P1{$oL7#Cl-+1I7k;P31T zv}ZitssfaVpYF3}9JUp_4(DhwT9Jnft97(EV!ckoxjDdUOD+FZyBvf_ahsX|>0g$95WO2>p`Z zh{$F2MVP5sHU3ohLvMSbRRb+_(m~}f(V}+_9>F`OcmkDuP7KsiFd^SR=@p|VK#U`6 z19rSbLH&=Vi1L-!iX&%x4&f#bsF;M)6yfR&aE`A<+;Zf6F3um=0pox+b(7VD7CkM%CuTQbar+%95+{1RKR0gk01*Oe zPN;Y=b~jQq)e$|IFtA25*|-%v2(W^g83?N+T;B>{*DlDqV`G~|5;=UR34`t$5K$xt#i?(Hx6Ax(ylP#6M@PyXyaCnDhYy{fI<>z~?H~nupX+jo3-> zH@}q>)o4|MjGlJUD9^plrOw+X%q8}dJ}`ljD_KlfVh5~`;M<7N0Vg#3j=jQ8 zQ#|Z?B$Z0l!cqLitYty*gB&e!AA6$9tcWVgGDN6=ayYs5VY+iHo`*A-51bNX_BZ6VR`F5$mM4$pC z1@dBFX|h76-Tn1z$L0&+se0B#@*NkiK;LGRR%Q$c+XqN*{R&^DCl&(^1(f)@koMqY z>P}d@A`FJg_y(R5ORVK9UbQ-^gW`@~QO@OWpxU5N2>*V9j2)Gk4aq&jQ>xb7-B;58 zL{m92kGruzLE{s8e7r@)dEB)(CjycTp>)X?h(_5!OuE8aiQ&l>GQh`IdUiN76F@G+ zb4H{S1sA`?r#~IMivWrx{HYZ~p>*!}GXCV;_g(MT114_cS3B#(2UK8AaenIKM9^Pk zz)IAlmBsIx;J}o-KJNJmZJ#@)YcsSuntI4_T8XrtP_bh7q`^cJ9j=sBdTGLrD14Ny z^E`KVtE?Rm1I*w}J2+Vdreo0y)&g5Z6^#=x0R%Q=j_u))%D8U}N)Q&TFR~6X@WKv0^S6 z*sB-VG{z|HBEu3CVp;qTu)?BD3kK~f$rs`RME@$J(g$fzV)1vc8XH5Ld_>hNzbV^I z<%5qB4G{8V-L&#!?bGu%)iHrF_edKrCocf@8MO|Mb)J)vWGTh*sQ_{}ooys_0$2u8 z+9&MsEd(2tEO2<^tR+~a6Mr-uSrKaEyteD#IbQNc{QRni-g}=#=8E0sZ6qZi30>^3 z!M}go$%$~9KF^P9R*-o29AVl+>-TSv@iv$}V0icK)bq}cFkzsDBxQvvuT&vwU`N<~ zI*-d|GnIFtYonc0YFQETR{j$E8)J`eg-MKvfC#krWauYF*2m7_(X;tBt~)xwIIv-?N0l2X|vg_cEE%KtUzgu5IV2X4BBxRLo7}c+!aYG@?tbDyG1ud?4!gR zjr}RAj}kFKtXzm1%sd}iqSDVWk+eRk9?p}EAp=V%eCaPxB zA1^Cb)A`|7yl!jw^CaMo6=<&)n{8kSyCe(!J)m=2S1lx|yO^Nz1DGh`32dknT9!;NlTmRfdz-{52mKLfpOM2|ph z_%}V!IPs6OZ};M$=Zzoe@HL0y)i)pXp#!^WW`Hq!sPI$L#cy6J8aSY*uabcwXP&mL zmif$Wa|Th>1heyJhcRp4__g&Sfg$PepMD}5Fr)9lyD}?TzN)>FUAlJWprJ3X9iK)=rr%XEOkMMw4hqBuuD&RyLQBtR(oKySzo8QIb z`YoylI&gmXHzua>YCwTLMWra9EHAY~TE;5FS=?*_N z&kK^y+-AA3!QbqQ2Kn?#^rJ;|U=sGwqf*IM{;rM z)JVwhs3~d=AilZgw*UU|grJE>fY+Z{c~Kl`MKCA%<0X!4C^qLW)E`Ve$TMiuOT+20 zJ6~ccv{X4O9EFfG0!m=-5jZzKe}09Z3CR_RA zLJ=QA?1F_QbfS)%RZyhr+ieKIrXiZc=c94)~ETX^&8^f@tH)m zHU@y=sG9M8s>t0vFhBV$F6SF+TjTz?R!zeU(;um@29a|wVEc8WDzA~pX90K zS`YvS92pz%15SE?&Y2=bm{Hy2w<6Z@mu!2jd_IDt+kYRK;$$=s;L1wiQKfHd;ZH;X zpevIIjuS&L2|QLLQcqo_Z$M2%If#0}X+807F`y~`(P@_W*~%0@eimsEjr(x_H)m23 z^dVRVRgN=98RO5CPDmuzK-$yyeMNqN(JYIDouh4gb09SJhE&n0s;bo?X3J#|lUzjF z(~|nRM!+coEQ1sDxp{5^k7V7RqHK`jM%a@kA4V4J< zPtGrk2vZ`rWso?1KpZV+r-1VL)EmheEARgqdD&MxW>_CMUHv&>r}c{&{h#|QKGGS* zsRio@fWEL$d_|n0|EWSMxe*-lKQx_XLsiY!hxa*jcej+3G)T8J(%s$Nc}SI%luiY1 zx;qXa-65Tl(%tc#=f(ds>^-w)*1CSz*x%xzE;qj3!OpR8lViV2XVO%)MycI#s&Pbl zXk>Qk)6|x^pyGtOG`ja;XEp!?#g;o}MRSDIs;>IL!RYIn9unm2PkuF4OqxJH`T$Dq zhIYT8pPJfS;++&O6<=K0fGxv`3DGqtB=jT9ZQcf>K!N zR?t5uhuEVz3$_3Z*gtrQfrhMzRuII4YWCMiU&_v}$i{SdTvG=#OH}+#Oxv(C^&T!9 zP{*fGTm$TKo?Z%5fE|F&T+`_gH-c~xT3DtPslO8T(eQ|x-$1pi@xyS^yr~#)gQ$pG zE}o8V;QZiQhd1I#!8|ywI?So`$*d#)^*cRuZUh(p#=$RRo~d*FE(jw-N$2&d1Y)lx zv;wQ?ZcZx4^MpNZ4^KLci>LtPJh5)Kr|aY2{DVbR>YuCH8k*R7`gz(U z(}q|F%A-(h6#{U~3a)l`&uu6P3HujxE3RZgsp5=4wYL;N!uBBx4bNOz>OOp8MAoXi zRkF!@qI_0tPCun;?_=(^A9Bv->QJE5JeC~Xy^t2H0)dg$sO|>8|`18ruc)_yqZDvEto?671^FX~p(~RW@fs_x$ z*Ppaele(}fArwG9Bg8H2RpPl?i$&aFvy6tqqxL3kM9s;ged{5fNDT@)f z#6zR{8fZfS!YemrWlsF0`vVj;7<@q1)+;_Eibb?!Qo$gghhq2UV6VEynCBb>}KBo@!w5-dnhe( zpO)}YjJWY6<0H>2Yp%iIF+cI<7hBE8N38^h^jTeoQC)3XdMY&ZZpoKZIIGU*cyH5d|qTxxVFW~JVo1H~^T zXJjjDvnAaDAWD9s^gWTBJjzlvrr;Ni<3;u`;Pj~z44m-$T)HF6?F`DEcl{%uIWk4D z{owMSF*sI3ofmj7b{s=MRrl}$gBaQTY}@4=%cDtgD|WVA^;9VS8>M=lFNz#UrLH^R zu_eO<&g0T_-1{F|#~iC)0i*k@y|;ifqmbFZ1>f=%=%jK*P|#aRib~aa&foH$V}~4G zcPvViyCH*cQpb3JDgvV;ICBp7k31UU7xH)NEOMVwC`n~WOdy~(dL;@Veh%kg9}bek zO}O7n!(&GKs*^@dg3Q#0w_v(bK#POa1fH-{uMT356 zLx3UxL#sw?>N!wiU=d+zZ+HOQrh!nJO8*DO zU3#SkWT(vw&X#dc{l@%USV;31JmsDq8a`mYvZm~M9(G^@I^Yj(LnnTN9{>92tLiGk z4obmQp;2M#F|}@w|K{$hk=ugb<^0DrferW&*+N^4A`l8H2cu2+J;V`-$aMX?D0LGL zk5oYWX{hye`ltr=eOG-~)5Xm8ppEg7XzP)?_N#$`!Q;pU-j!@G$F6S%v9A>FFdb9A zQ~mR)gBZKXy}eXfoL!veEe;^QTFU8;uC&Vn(%&<4XZ~FB6U9W6fa0kIS?{vhnW*I8 znC8R)=+K_x*dQxdx1Ss?mU~agqX&RZWRJ|?k=)_9K0vZIbj{lM8f1Oge5N)^!4pTY zkZ^a%iz#Y=9#HN7*a`p$bzq=m$_h3pj*mE1vg-2?@w>4fRtUsOUF#?au#7bPi zwH5wH=mt+3`%6jF^i_7Yeu#90rf5PK4!$2 z>^#Yf5QOVNH9;FfAQK+uAn?suQ?nr67k{2)j7mapaLcl zl#~%%_t;srZ^8-Od>9_@-Qcv}wHkL_Jihp?8VLa)jg6BR##`lomZapuH;pfc1hQ8) zLXx->TLrstRC6qWmCm%oh{E@6G$$4{=skTENV!S$q(Z{MhK+){l`m1JFY}KRduXTg z7i&QNEP|E~?z)48rKhAT_4#;8eD6x_($=-wmFHo~o-q?{@o_}@33nRwp_x&hFsd|NsZN z%4ubZg1K_P!hcU{D+-_bdvaDayz=&1&tV{eyr3pN-#>MS3%sw#z_!M&`Py2a<;FLa zo{WoK!mRTT`PO5;gdZ)rd+Am^6cYS}3T`Ty%b7d)>8JNN;Jo*q2*&{q-=XZIETAt( zYBUUvjQ%lZ8EI`%&I@S@2 z3whs2abbEl99x$F0%B>Yn=j*HxEkZ|1%AU3cR30Irv!j4lg8;^A$X8)D%c1(s96>k zWXh7WEAJf{b3eAuJ7WW=!x!FFofOZFhrY++%9pY(V(6b$Ofq-l_LTb77$N*iY-7;` zO}yFr)i-I3jK@df`70IlB z-&#$z>NKX_V|`YHizy!p6T?6z^Srg-KxLfGpob!edtir@*QFt$04;;=kpQ;Et;aj} z)SS?_xu|e9W_|m76+kJBh|e;~Nd!&nTYl$HJYiyxi*|N>S-$@8hrDhqAP^+_6^ujy zh2f@4C=DAybJ>ms6R)LqL!Lnc-*bgh!wq1hEJH#rrK5z)Rca?9@Dg~Pf<0F@T4TtK zSDqAM@OP)Mf_gKNY1z@g1g=fUX0{UW+z9!86Bbq%BxN*J;gq5JFRTWB>Ychs{W~{& zOa%f7cFVeq8MM9UU5gb1E8Uqb}J!OC-c!49_} zsN}Or$KA3<4AvPM@WF6M1T0#_uJW0&2(-AshDPrRSDa;xh3gvh&V>{3a3#} z`BG6V9y_vV>h(hPHe&-0*fhi^&G;u@a{d9)L0zg%{zetCSBzffo-GCXj1ssk1yCFY(ljj%ofs!}_#-{+g=PKWBCVq*i=3M%Miu@^v(2N|wZ zu{Z-1pFXJb@~pkkH3;C)g$C$-#B2isOB1>=fNM%qQX_Bh2Z*da0A&t_!*cp}5-`;} zfZNKgC%qG5Az!?Rjf~;J1LVg@@P9fAIUYmE2md?Hy69qygznX{l4@EN1Lk;5i_$xp zen0JKg@99tHGIeN-bN+0)B^ZaYskiw0z?F;&h%|XVR92-&ky0FX2|fd;7XR!S-|0% zTANLE5dp4;^8v7h#HqsiAGs!*FHSl2{?4Sy)3`f%?G0xXkbmX^uze3e0xxsnHu_9m z>J{XlI97&^b-A5y7iOFhdPBpdN+v7bCNuuk>M$nNj4o2VrBoMe^sImj=C*`-GLyQ4 zS7iLokaV!0`D6TnpAG+EE*hJ+<*PoO!$!wS7aBS}KegP2mtT(ERGQgK;sCE{DlB*bSC8Xo#WwP6S)#wYh;$pV6KPGdKR!fDr@t5ch6 z>#q+v2@6N4SI14?Dz8xOsTrmsTZhnFzJb}e7^oXElWQBxX5>=x2bVc$KxIYgnJx)Y z)Aj`qUp|Mj#DUksQl~F4esfxtVDsE@G=K-J|Do%y_XsTqvcaPn(3$)gR&E%VK?lgd zwN_OEpDO#N9i&xY@&iT&DM+`&W~9vdIdm+YM>VGrGFFKj-N2a}^Dm|}nzr^mfhvQ` zO5jQpZUcc}pn^KrxHV2|`k9W9EE;}23a;S&g z2@kPh<+v2;W)&HE(81zYk1n#iC3!-TQ4;qm3-eUSfUY0TR>DUCQvwBE2MgM_tHF4s z&RX8Ly#&~7q`t%bu}Nbg_nu*nLK}eae(4+<9+8PNRl?q_(gd1z!lMRWS*Gi~E3 zkPKtWRO*DH^5?~KY~Wvblhl~rbym9vq7y>WdCV9KWb*d_VnLw4Q`%>*Z>cAZ>`;>* zQ01C1yLC@P(86yRbm3E3SgDz4-tHSk0sL{G6fEMJ%5h@wvj+GCa2orUBe(#9I2^(X zcoPm%kd_=WL!cPC4&)<8R@J4a^?q|3F2tECg+;3Aj4r%g^8N09!WQtA#bR z>Mgkqldw{*A={vir3$%V^fi=(aRm$l`0|5ENySim|DwykE7q76%Bza7h1$;?YO(p* zO_P&tEd+!RF?HENh~T0gKX&R*PcD|Ss&<&}IoyXC8MSxV6vGqoAyk$q=2EsgxI?KX zhRpzJCQ@(;L3~;58!Zcw2^iR)BuXHRKHB)`cZf`TH+Q!Vh(EoJ zB6B?)>qrNmeLnpziJx)~XD{WwDeVpAr; z6!l2rx2_6xvA5oON?$0eoj~~vB8c_2kB*r1Ti6gZ26-UnHWGUY~DpV{_04d z9SM&~m66D7*lGnYsV$y>c;9KC>cA}l_cc3OZ68lqQ@dI5kDBiwJW^dciDK5m)C!$@ z?Rz)d=3^(^ew$bfj@`cmpQ8{=Q!HQ!q8~ zt;#I-qkyw;@E2hO!Z%LI(HKF@h;30VTFnz#n1X~^+>n(O=OZt>U;60c+UJakgV92; z9bWlUZRf_0I~amFp4ZT{@uU0C#=TN^LcGNkNT_pTdUWB47bD;$=(FmGPeV}Vb8hVW z*tA|8%>#vs9jzPj;=ddpOKwC(fyJZ$F3x@n6h7{P9c_FPJaf;w$B@g=_x;Q)O~S!U{cv&BGiUbtSO#_}XRr7?U{M*t1&ABZN+Od2hk z2slU&uzgN6mB_uwDcM1oR2_aqByD69)7d&T2Qn#H zArS`<+}2rVOX;Q-&q=BU?dhH~f1eKiJis$WD#>q*cek@&OoC)#U~d~TR!CG_+|=H= zUzI+LOq8*^6zx#+uiD;iJ1DW82IqcFeRR>b0uK^x@}7rY}WZf1~Y0`AT9#&y*tkF~#Wx-u5kg2X?fZzWu>Tz8V;nn>jN-M8j+ zMs~E@o$eI03oW>+8VO*7zE4T8xcyF5rs;Cvq$SYx!^m0H6F+72C%`@A?zQ~Qu9K>OgO!zlits?6z4AYp^u(+c1(>8DQ|aQj6yEc-lOs~fS{Y!aDng` z)PUS&GW#=erU-dBd|ei}(Gi*hu5;*F!inz$Abtc*9>vg6TGQWQeihNBZ>SLQ#3AgY ze(4;casT*_M9M$sRoXlO$l32m+}=1)=Q_Vw zD6VYY9mkzx6-sy#do}h=m&RvW^)u|Cd7RAG%GolR9w50C7&4aS&AuP?^dVdEOXboG zrDQmmZSl07u}E=8mEgxoIY2jD)b<-;P@aW}+Gf|g`y_sf^8==5Nvwc4DPRgm0tA47 zu$uh10#wuEDk~DHiGW4K1-A9WvJO81Cab^MhhNc5)5LQ-;R)5v$Mc=V$s3JKF>D+m zYLxgUK6-#RD$rJeT8TTHiEDN~Ax9#A;t40N_8)D)H@k300i|4uOXnZl-+vkd&>_W6 zedr3#H4#Q$a&za!F!=MgHJtnfTJq=Iv!A=17-aJgF)L@HZk~UU?j-=MudsPEpFuUh zSi7sOKj*+{cV#}B3yebe9THr|Q2j6~^iy33K2C+o@;dvy4{1Ib(9$j@zy{d==6EQ| z$WE~kkElnnqs;5+onm7c9b?l1tN5gINrb1|lp*}8wDQgzWQVo6uq`Z^98{3X@&ied z*uKofBM;aT)_F}(GM879>q@Dxx`i~~>n z(kqZL)H{f2ZCX(l-EI61;CQc9t&^Sdm*Ok>huR&HwiZ6q3%0tS9Ch$YUOMp`6m+e4 zz-BD(yM*It9gWMuc=<5%fnWxv^WxP*DI86n1FvfsI$ibA!`tH|b36Ge77p34#gzW&=7ek}jXM(oIMBXveG7{Zi^2I-!8@fp8l5Vou?HSekvwO_ zVFxu6e*Y*D6MyN1luDdEX!?<8*D0COfu+>p++6qznM^3nRA?vMvZQ~ewu1WAA?fDss#82<)rs2hEF_QnZ z7y@iI(8kFDZtaQSA znr6?6Ayaa<1Oy@Rt~r0YJLK$yPx;#)9lg22G}O+)mb$5i0|A(s+x+vp8ds7kbIlYr z`(KZ@NT)3BL%Qv%RwCybM2ITWF7OhnMiYWIIQ`bKJ|OpQdcy_0I6mFhxMG0oM07S!#BGDvZsbtsTu25FZ16L2Fsj>$V`z1_i?6P?k z<_(#TS1YYcKQL02$u!w|4^P67S4@q{p@PY{a{s&*7F=DpDfEMg|0ml|0TsUVd=Bd# zT54&+wr@pKag9dUXiUyYvM93D()7>Akx-P9ZzgdIIBSQ709?>|yo@zr$^>Q&8r71V zHcDU{{H6w)B`$787W5N1Y#`dK2B)2YdCs9b-vg9Y>-FwxU2b8jXajtkjWj5n^g((_ zLVBZk!l{Uhh)-7zDdl#>G`Di5<51X_K>xCuiZ$%^Tku-0R0U!^%e9r@U>!0r*NpuVKMY~W?C+jV^LS$!VPA@Vt{aF0uK7Fc;5H2+f4P{ z^@oh<6@$R#4>_v{w@SNh(}9ra?6nm9%oF&Y zW_^rEYNA*){eDlk6b6>?j%ByA?>RR+J8pXpRdm7?%&>=ivO_QEn`VS8@pc(P*kj7Fi!dIX~^&Rb}Z6$gp$t#~#g1_BYB*tWxi*YWe zvze~r=Srr^1QNVAN=l^32OLq||30XS=kLCo>f=K1;n?IuUZenslg8MndqeV3*En#xL z8X~IP9J=Bst*0$sRKVu(vkh0ZKjyci@q&tX&c^EJ2P%#W^mlRGNfB9*Bb2j9` zjDRX9=IyxhY9HEI+>ps{c<)~DxBF>?Q5h1Q~NB`bES-J5qqhYLz4XpR-u71nUGr&(8uvDf3wCc zc{+|1H{0i%KTiL-z#Ib;xJg6m5EqlU7`4}`Kb}l5k?wa(UQz3JJZ2a}v@X0nsfirR z3cLeTfaZ`WBcj>!nZyP3&+7};;N>ZRstZvd$ff{Hnd-0psCQzLC(MCxxnTLp(e*d_)s#_mf^?V`nMCzAykG2}j~Dc+u9& z{by%-!Bzeh=;5-2!YL}anN8DcEE~c_^W}Sjfh%Fy?2uJ};4_k6Y1l~O7!^i-r~hW* z^FVcQN#N+V4v z-8wlN8-06=pX|lqVy7=lQF7TH(30?54Gh21g&n^>`Evo*+@ULLuDW&SAg%A0vSR2c z1L4Uqs315DsHgT6l7GCLjzh7guzx*fXbjvJ;lk_|_MGE zV+$TstoLQu`(RZ1ljRhgfeeevVU%xG;{D`?2&ATwvaws)DG@MOBH<2M>DXuj8b%Ww z_J&L8+Ta8OG7ql4q<8leEm)9v`c}Zc8g(~IjwF$mzECZ6^tj&~&2iFe9tn^(&BQbr zliv$vBQPYB*_+}ZT%JGq6e#xj(RX@*FrR^i89z?0E7~>?Igt5lq6XTG_|YA%22w#F z-*&WRWOIXkc{PWmC8R`cppi<6k{spYbP6kfllgt2SpOQ{%?FdZdDPS_c+Jjax=_+Mm|Pf=17N7g8?X&Z^ReE8cKFPjQJ8tnU0?>2O+m#FDp`=P%4Nv z>j!LDofNrwoi~_qxBN9t_&5&{kM0y&syV3+iueLQwa$vh7%f3D^*})1-Y#pA4J>uEFFpIlZQvX3pq=!y~ z5&vwT`us&Goir!`^Xr6-tx(kPPH(h~N1iUSDI8l^-m8=B4pE3TApwX&hdAok8)b!E zr~hWiC`IV3quFooc;*Uh^87cWgq?M)BP5uzyc#K&;^zi7@O(Jh&1}+1w6|+&NaF+} z>Lj%@YwUk=Y<{RJRkk0`{{4)8Y$jVgMJ80CFXz5Z20(py?1G*oLBz$JYd$Y z&_MI0w~bD(1`&R0`gsE7AV!iym{2i)Dsr`0#|0+72h@dBX$>$S%rSP9@9X=7RGAx+CQ)WyEvR?Q7wX8rW4jYe$ta}0Gz>OdB=j~?=V<%H+Z!@{B! z2WFoKoYA;_tY1^*H$ux$z+ps9bU~yDA2(gXXA5cn43uZ>l#*tKz(SCxTL%lb^WMns zR}k{or1e3jPEV@HTC^4pm)9lh)RTVmoLV?+1K1$@NXwSvAMxG83D^~EdUQL1n{Y?& zM^B=RyCWZoP1W^S{(BLcRru58Wvg;>aD=1#l}NlUl~x~^hed`5DZbD{l-)}y4k7k7 zS;slC5dgIoq#Uc)*1t0zo&NYkyL*n>FeWo_Kz~2p?o)wt|NH2x3o3P~0Gad{d;rRr z?W>GKaaPC)A#jbyIpA)$gOip0i({W;D3&2Qv;Q1#yr}D2QA|PoE2JG$B_Ozwz?j8e ze|RgPK8dwhb$H?5SoHHocQ9%NC|+xN5^Q>9vVyYD5;xZfd}x}w(QIL1os3-YSJ6GY92eliYYd9M*;@1=Gf9p96uTf)SE zxDnw*>B6RvaQvmp#^e(K6@G?@1ZNheTj(oD_8F(Aa7BM0nD+b+C%ma2kl=RRuB&|$ zLA^ah0>+BwK7oin*X!J^DJB`R%MV27r+-^VM}6l4-Y z5$6vZCf?!{E%8>=gR%xTjDZD5)C_^}7Vmxw;dw>`VEnnlMiQ6`9OGP)BoHT* zqaXk{RG4y_fBIsn17=uXlwQn)1-{#ek{oL}A=lFM>P5P-yY7X{_!w`5~0G&&aT>%#l++ zNr*t&$2;`DxWcnh(Fg_k?cf@actkM!(6Z-o*z@+=gLh~ToP*v8@gL}C3TGn7cK?tl ze^c4KQ*+QgMW~SIMPsw^-MK?ZpuGa)`Cac58+LvR+PFG0L!VJsaQ9G*$t{%}NVx2~ zHk?2)L+K>3wHLN26%i02uXxobtq#XVn@{WOiP&v1!b1UGwPQo{W!sx zZ`Cf;)!}Tb)V|VB9;SIg6l&{7PPgL(B4ZWAs_)2eUX49`Tl3C%MXS${;x`b1=&18y z2e!d@f9oz@65*%HzG^D-3I$We;(__yNt3&vFh}h*$>kg^hqQ#5Y~t8*tY7k5NStn$ z32$x8XZQQ8mo%F4r8rpGLOT^%*OC_mi52M>ksdEZD3LL;O)Z8 z>YaY8ZbJ?W`D3Fhd@%HLr$#l*)gcO-O^1csYA$}M_h#alk&z%u%Mo4NlNs`#MKPJPyp^d@iSR)!;aqMVQt$1TG_IqV zRqp_a*YioSID|lHyH{q_s?^((A3t7wTmaKRf$OJP9*MYmKYJB>)~6xCDeQP}BN{7@ zO>V6chF87_H+b`F{Q+3?zmM88OCOnhl{+G(3;4crkE{NnX-WdL8jjOCmzg80L(j|} zcg+O`_=B>^)nD}W8LNosQQF^N{hsIz zK|DKe3$0K8e1#a!|eB?D|;5T@%F-+ z3jjq;Uv(&(!a&XaLsd(y2O*=-<-og&eMCrX z=`V=T@Z6I<69u&9#-P`6LD0AbGL@(x+e%zC7%B8?G2|P?3JWkrX~QTpTv7-aZyyX| ze<5e0&1FJ0CJe=%bwCrICtOx|6;vPe8iQyjD$NROsD9m$ktEvr#qL)qe1U>vgl&tA z2FyPH$3>u!=50NiA(GUl(wBu8mM8ksue6ByBmTFGq-;EvSv;$E9-qu_Nncg4S?|ch z`4*2uPwdF{`RNef;<#28p#x7Z!cB7^5a85iJ5In@|dt^m^YnMWt9q*J=ucvT@WO=YKM)|-W(s6Nn*^pW z6(WHurs=8-hV~9@kA6&qBCr<{IBNA%R~IN#?0!C%=F<7D|1kv}^mpT9(GoYnnhXGo zu5@Sy=eL~fWQ==EElB-k0haO7Y4y0Uf2GZrUq)gOUbxMHSV&0iKRt3#SnDhiU2gTL z61mLMlSxd(^K-|Y@`u(iy~GyR6g>3>B;@hPV}WgZ)v@^l2jnnCI$Jjoxz4yK-Ik=Nz$sn0nihLIh? zjz@|SyLyYIvkXgro`(9Qmg|f2GCo_>>QOD&Rq5xTemrRWPZns&uC#qn{qI{H!7? z&^&W>?n693AXfzM5n{Z4A5|ic8yYN;DQa*SVsORJzK8}8BbEmOgcW{MZ4RK{tXqX* zud_(IMLw*wxpw63WX|kX3a*;yo_f!1_z)Zr5KfxKp**{+ohvVQvsLPfWL&Bckbn#v z^pdyhCiN~*rDRx>i^w$~uBVm&MSIxRjDJ4Zgb-6 z&76YkD$~$rKFB*$HrW8su7Wyv$hjH5MKr#yIHDWqE1M#?wq_9W&uaVDrv2ZUVMyX; z((_b(d?6*26CwHw0<1CCPbpdh%)#ZgZLdG#Ig9>6Rbc|{!;ap>rrB`Iw^91Nmo_fX z(Shd{D~rPQAFkfcXGdnb8l`p9@~sm@4u3ip(?oDclxwHeNl% zQ)0@aDEMQ@y*hZI8BsW|X-0n%Ch3D3x~Kd+1(gx4h0nS8lkUEP7B!X*iM`4GYZ`QU z)}2ru-`6RP`)U4OO_0qX#*Qi9Ki|Fo{odlJ-CV%yQ){~5NEERA|1;&(Y}*#DYs~!Y0Y3Tmt;z3SA2ZbuP8Pakp4W+Ts&gvG}ft z&jqpsoHT&Vrq5wv7MtRoI}@K&jxVWyKvhW4V9+3~>YJpTNBP==i;s>J)@rw^h^s5# z94_#YY|N2qU|$skaq}N+KTQgGoe<0f#t;5v=?%u6gdF|APdyZzIvb#*%Pu&s2Ij?F z+6T?x0QH?DD`)bIm(-#wE^kMpPiouMY4lyBhddd%M+7{<2&Yf&TbX<%JSjnj;sBH# zJ-Y+bod&9)Cl%%7ywTvjS4Lly*)oR#Amj3#Ci#fj8K$w;)2QbT(O6y&k&qy?FCp_` z@+)!E@V@U4>_XUl=nAp%n19sh0ic2l99RD-Qb>z#9YiU`R*aZ*Jm}TgeAzwqA*L#Y z#`AmzhdcX_hzVoETWY3sUh2 zB7{WLwc<=CWBe9Ocs#XVUuFk)#9-FyYjUsH2ZCTYpax%%1w`q)N!+n!qwr^GsT2$d zJz8x{^j3c8^#%bU<^<5J*TK78As>Q9wEXhTK(sOMg4Qvo)0K2oh!$ZyWQHV zQ?Gc0+0q6-cnkEa|00>wfNS+nunsE~qW`U0%DNZ+m%r%xMkMOvXAly8dL#&PmSMsamBhy3Llcl`;CDrx%u@U z`;hXN&5Vc}QtLXN?N+|>UueS0F6a4-b#2+cC~csZo1awcjF%?skYI*JqZ;@VMs5OB zSo7Q}GWq4B1k98u9yOOHo5&Vy`o$6QJuc0@1c3TVbUaNw#{<-UZ{XgHAyFhW>b0>? zd){rI*&A*a;-3HFr>p+rq|>#Y5QcySi{kS;pD*Dh+3%Y#Y)Qklg0fhT>a9Y$DFN=& z^&83I{vB(709Cfq7;JlO!#SE6xjpK32?xn*VPXrp%&XQAG<8D72LM}FvgCiizx~ob z7z@M%0A);+4zKC9=-fAZL_n^l=kob&e|#hLY(qx^ccY>HiPZ2KCa7Zz;`9D)#9ar! zupn3wQhwVkZ`g_gaQqgk{HrQ)!NT$ni*s+cBIQ(w!6WtOI$N8ryPa zXpwZdFd)1U0lHTK$vcK?bL2Qub31=}Alf1E{tq1a43-pD z&CD8t=YTH=l553Ie7eO1K<4ODn|6xn|<_7sO!Bl_Gmtdd@?KSJT#F05A^oTzY zKsg@6l4G2^GW>mJwC3*wodsKrVIP; zpKhc?0No@;lONytUl)!L{;YTlvQbSSEC-8=V58{_{0={-94(M=deUXiWL=;6*FNBtm%4~GQ8z}Sw`A3#Y>`^dTi1VEFnP4Nmyl*U z$shm|=wRc>izBNW6nh8EIZ@ZEo}?Q0OB#f7u#d+nkgm1Nhvw_ZPe)J6EOLWP%~T0n z{li7TWggU1D3Tf$E&4iJQ zQH!kpX%SsF9G2usOee>3wCrIcWNRhPVqKqdoBc;OZ#IHouHA#GpY5%Qi4)!t1jq9Z za9boTcN=k)3b4Jyhod!Q^68AS{evxICUgF1{XC*sOi4mO=a2ySy;_P?+$=d_8shVrZAE8&4!E5^DjybPr- z!VxR9Mmk&kHPT=zpr+b1*g)!?{8w20T-LMazPk(vSl5c-wBU*@cntm@3gv+Iq5LOx@SM64$w3PJka7Y*9^gg zWWP9ySIZRG;v+FB>QU+{!QF?})^@mPYukktb!m6f)}(hhH{@eXpPpG4U2+$%eiy*! z^mQs0w@efZlw=@OTo%>+5tLQC2oH?Q`lq9+oE;zm6BbYq>Vle~P+6+kq{Bm8$7Nc{ z5L&RawzXsEUK69s>+`sfi)Zs8GSKxPlGxmFLIpCnG4-TR4agZh9~evuw=|&#bV*`{ z=J0M=FgT>Z648?#W-NgCs2l0l*nv+E2y*5u((1MM3mxdF-4VbnKyUXrTHl3{S3xSH(9}gHZ#>UEBtbNIFRyn=kJ>L1+lZ5+L9Nb z`R!J(eNZGbFl(fhSr8>1u~cfRc4iQ2vIiNsQMJet(K12^46_ZKVsvKKsi4nBqPgmmTD`t-t=Cymck|acN0oOAEe*`a;s5P zcOpKA4CbaVXWYA|8ZfYHj?(R1q**-?Mbn3O1U|Ja4Z~8L$V|PO&=*RQv(QSE! zjD>;-sneUkJO?sRgR1i&g7vLqOZg2cLNXb4lV%g`F1%*sf;^ ziYN?#ohXb_R4jndO9&^5PwNty-rcNZt4_;t% zw>eOsZzMnv5fDuGR*DN(Idy0?Yp#eZs>RehD0QxZ@P0540Ep}Dj`mIgz&BMaPHRBN z_QeK^>c<@JBTF)AY%NZH@UB%I87A)rPXbKT>Ii&LQATZ$;acIVci2`^pF#4Gr(9}S z9t@38Y2#d#SyO+6f9^Q~Z6a>sfVDg@tTl+;>Myju^Q;9abb_*VD6gr#_LElNfsM5A0^n0C(i->GE>~U>i z)h}TvYvVG68j+zk3^vu;%u-9U#(5me@8F9Ny!lI9=Ac_dNvvN%^&hO( z!_=<^#C2Mi|4xZ#CMK+>!k)&!W5E@~FeY_HQvKYIQ_Q!%h%5{Y|3-QEEo)LL!Q$21 zG__Hg1V@1dNp0WTu$}8lX}hU`4EV+809mlDc6Uc(%2Q)1j1_PqFndV8H_s5sSt`jT ziRPp8%5-1EUfytIv|TEz+jPxbY>cb1eJJ7HY?IBJQhWd15>$)dk-*HR0x{a(ASHR? zmxh2*7Gdv(G=0W`#ZSb(2&_a#&VMG?L1k`ZulJx;Url($SmfZ@Ktuszx%g3YoykY~-uja{^qc;7s}`YXniYcY;v)iJIR5E}NLiO3eIrIB>Z}d?e|UC5($bbZ36&wZhOq{br;-vo@!NXtTbo7E1wHch~Pd3}L`CTXo?Q0Aty9brf?Ovx!4)`A!@U8B- zAeYz$ul1ANi7^z7mbStZ1JBbaytNh#dglVXdyYG5-wEy7rJnF>B=m+rn_VVoa~+ur z$%XHhZ&3-REsKPsCgL{T<*1e`Ga8f*c$E2&+a5uH@zE!Ou*w%d<%xZ|IxFrw(Xw?Q z$~)Wg^8^*h271r)+w;Xkz!CJ(5o@|e1dtHmSik6NPg6JoTRT}k(?%p4rX_To36P%2 z&hSctbais?EmfcMH2$yyrg(K+tb6P)?1^_>XL%wtfCCMD|IpJdz>&fS4y!Cz(-ulj z;u4zo>!_6o80;29wpAy-q6=uG5{eJTc@eQl$HO-0;prQ$p6!9B7KcSif&bs zIe5pFuRZP0)f5m@0TJQATGNu%+=tVYxY}KiADBIv;(pq}0HF*+-pO^@)Nzx|rpCW> z!GM>nPZTEoCGsxs;UI=UI2lRY;6<0ExpTgbz`0ZesA)<^HQr}ragN-P_IqFv4QC*2 zKXeB=e4G%0ugw|HI&J3VDb1hr(c!vlDKPx=U?$7FUh0SsARyUX?f|b3f{ntM@Ii5x zv!%bx_NhHBRaf{Q030o`0ni$V8nuMl-!$4a3G&I`iJ8WI+qw2$3DiVCP;0a;=6kL( zp8P=&wO2tCQ>Rf{l{6$qadLU3BfuLTl!jeDV$imP|9W?g`f=^|eclB#hY(*}q%Pe5 z(R7tS{uoM#Qke)-T{NoQTd<0R&3!s?ox!p9>@BmnQXTK+8mR{W&l`L`khKaFGnb$!6& z1riXwk2!98gMjj7|MT$FYZ2;S4rdfXGjom~1rY($u-VmXTPw}jpLWLHZb+BBwvE~y zpfUTww&ib`1P&w8`VgXh)F_P=mN*iWtlQf}ikfiBO}4g#5?LNBs9Nq9ZuL!4!kwW9 z28!IT+el|nMegoE!T;da5$C_6N zVt{l!$skVEj<<+Y2Wu81JQHrj=XyC%Bz`6(iHX-zULil|7O=Sboo;)a^oSIk&>-I| zI|ZYz`_bO8qJlIZ)XS^AkB^PVnElmURk)p+xPjPshkYB6%DK`ol*X^=TDahQPPRJ42ht&*D=tPP*C^GzO%xUcfmz<$J1n=W z6v02m=^4{3djjqmE)w&`rH}u_74^}jKq6&DTG*}?@u13VsfU88m5kCr zu_af{8WlMIJUSOxEzm8{#{)fV!Ij+0$C&%M_BpQ3>C>lU$ezao&DyR#;fpwF+4yf% z*pj#XDO=LCo+-_bgn2F&u3DvxHd|bhq_rJESEvLD_!vyU^Z5Q5|DKY3oei04I6|BRK~0V&|)29COtJ^I$c(>oe(-i)wJWLi%ScTM$J0T zN%=b-Lru!}Yq*jI(>W*FH$G}6EVpPSA}TxRALmzG|EsnN4Wk7&5`;*21N&xk>lW;c zghfXwv2CYC7B6#Ny>nG@V6A9qzuePNr9cDA!%h0pLJMHYbCGk&2Y4dB1bN?dTIvu}n)iZtDq}kc@r&ZobB)H7A%kXwoSzqIjcJ5%5LfFcFY?IFe&K>UG6Ns~ z*`dC1@=CjpdA_01g02P5{l5hsB)>M#v*y{r^9WRePs@CFrFqdCk|3FVb<{Ia96I2N zvy=Ps{P!qLm)I7{oaFa#7X1jgp{ON?`F>)#pQLm-g!*kjsX7%GxCuoB%dBOos!AS* zVhF{pzXwr=`WOHdQkL=E{+a_gN5_mwfv2tlN*^TNkG|NQe~!W|KU32{L?(5SpIAUK zWpy40<+*K`}%IohW+`YEd5eU)Ty>P^wv*i z_0dN)snq$XQT4n56#pq0 z{&{vLCal6C26X@N;VAvO7>pu{lN`)lbdVJ4VF7#YhJue-^U~cR!&?%4e2fQn^_DW^ zbwMQV61bhCcws^0lA#V4Pnrb!I@&}kEVFqF^*(EKm|Gg!>yPaL_vBn!acx=9R6gAV zq34F&WLBe|v+rL?!OfZ=S$nOK#6Mx_YF+F$H4N<|wKwbj5Rm&a+M>$kb9&1@eL6D6 z8WJdpHk3b2Dfo%x3|ifHa0N9zP`1tafy{pO7L z7qf|H)*b%Q2C>8Ap8Vng7jnQr!pCn;p`Qkz+WuamVA6aGfws)g{M62Crs~+?T?E^I zC?Z<~7XPcMs*>0y)hQPuGfrlHl~|n3wenaj11*jVjqUy7SoS&~Kxrsxzt%s*AK78z zRx+5hurp}Psm4n*JeAv{?@>G6k91Lp_12HCI*00~hx~5pL3F8bdfK=m@#@x%*c@inHV6!x z7cfobXa?sle93ZMCZG^y#rnq5PZ_DsxVbgQq?l)4Wr+D9)F03h5ZJd-f? zDHlcWu%Lm}I8s_ImL>QDSl@`PAmXEmk@4B^Gr;ZSevjO!2wT!_{8a_Pbg|{&G7q~_ zAX!X8LL$QNK>+VBnBJ!8pawi%Z!pNMH2y_ipNVurfaOsypTQ+%04@$&LO>X!OJ{d~ zG@7ve#zJe!wKD3o8(TmR5^^jiUnAycclWSbBE1OOAA)=|LUq$}R615igxLgKz&Go~ zxd`p?J8{KoqR)+(jKOb0$pZMB**dO&kEiU4PcuNG%@M2jbCdt$N81Q+guntGzd)&1 z_wVu%1{q@vR)m?G9^I7CEFSV#&On-2UzvM#RyD_eXKWcm4oT%WMORhm;GcCVH8}an zrHOA<6#ZAWY=V`dAduFCc-CQ4JCDhqBZSu@=@Y~J+mb^+`frkmu#zcsuYM;zU@~`Z z=U>7{h6~nm$YBf1NZ{quFVzCp&Z8IrR1KxQ7M5aad#^txitfGCJzw%e%QLbeW8v#a z*w}L3s6k>ul8QRpNUZ*j687o`r1fp6pcu#wznsfGb;={C?-e~0We_?*%-$*7`uOwk z>1gWC!BIoQB8L7c*AJAzZ16`4Sfu8PDFcLr-wjbb*Eg)Q3 z8dLlj<``oH0oeRQty=|OIj%Kqma5S~(0nUnzXL8zR}1@rwf;{79!R1i?5f(StD$p+ z5EK)z1#Soi@4VAx*;*9)XaMH?+dgkXOwdgJgD)T_Lr$Uf>go3Riy#;#0x0V?tgdpB zvTL#CS8m0S>x^`Q1r_~pgBM5eV33kU1pK#-tkA6?Qz4;+=9s%TiU&POYk2w+9Ci4~ zkRy5KQq@lL@C*&J|C=E7SXW^YKawX!FKZKO>~Qa2Y+_g@oWEo4A@{uhA<%LQKSltk zjFo?l6qI1|&gde@eoLSM_9vy9z}Nt#{MXthAv`vgE7qt^`$I#_-*%Vu?Vv;uZNKzI z{Za(07Cq^anMQBoY=KE<3(G^a$iajLL~hXSrhI2coLi%7g&sQMy9YcsMgOQ?nU$_& zA>S<$*F_vhK7akqr(VIOAL-41L4JLy?!pSVoeIpsRJfSHAogBW0N&Vh^#TIL!;jO# z>cIlb`G~l%XY$rw@nN#8vss&_*CGtWI?r5?S_(S%0DeNI9VNM&n)L34L4n$6Ftzp;{%Scm0Y09%)+i1HF@lRTkkH1X+ zzL!Mf_qR|)rkjTrRGN2$jou#FR%hn){xRW>*Rgl1;^!Q=B>Dm9C|9PFZ-AKR! zLwm<{Nq$AdKBV})NnRcn&+DtTHA64K>t6&M)>e*hNy{4m-1Yo z6H(^?gYBqn1pLDu8k){iVbmoFbz@=+c_PZD{od04&gmxXimK z248=PO3MUvp6Wh^k59esi9&rR80h8it$)Q0Vc@`2m|17t4ZclpCM-u9;p|z-RlZDN3Q&hChd_9MTKfBnyHBE0XlGAS>r!cx~nA4+V@ca zCc0dxf3SOk2&o{2Mbf#CzlxB46$N8pmr`1Piwl+LG4%ME`y!q)3?B8y!+9c-fSc#K zNHY3JCJ4f-O;PSd(9&q?l=I5H{wcc(mOlr+73c1)Vs3glnD<9bG~%S1YC$``BUU^_L7WfY^HI#F4iHHvr!cX~%*c!G(K-M{;o-|%r)xoZe2 zA*1K#3Q!g%pDZREek}YgVMF7iYrabjT2HofdzFt4(E!magkjsp4N0%2pktz6dW?&# zV7-$Zuk)=*hVaQX1~~FoA*jX5zlqoqgg{>^mtP9|Z>juJu8tBo)A-ij zZ26B)6sVU00)(nAO?cxQ0d_7wN%U9Bm!1DzN-TKo&SJsIzsN!htP`JrYKQTiAR z9-a_~IC7GdQ;g(!9avJmK?ZXaF4Ew0oMOOlNj^GGRA>he!GFD|O`p!b89x+m{e_%d zAe^uYe_1rD(wmD3K;{7-ey)^{!m`M;8xGkKVBQ>dc2ku!!(^;r zzmCD4wvh#9-x!fxl2qX~ux~OLDSGRQjfw}H4{y?E;Aq|mB)sw~>CR^qFIjusj#%+f>t9rN@@ z8zNW`B0GsviLMw56TgunmoE-We@dz3JK#HXdmg_I&w?K-EKLVL(Rr*L=Lw*KsS%5p zHY_D6`)>xk`RSk%UwL=87O^7&o{JsW6I@ID{ez3E#;<*|O7HI>u+723C)lK3=S!Pj z8M%b;<5)%bn<3K1=iTRNs>EkB65#S^xd)GxYa5Wo1zA>5Xqt>H<-&b7fN|V&@X;Xj zl}_Qzgc#SsCHtt$Y@0?bZMrExji%o{{74!~$TCiy!HpT@%F8G@=STZ(ytX*Z=a8eR z>hm}6ZJsznE3z|pYEHGW7zCT@-&GhYGVm1cd)mI2pFuD?Jco}V79m$Sz$~9wQ9gFK zNaAYP=Je|q{pGRd44=`braKX@uvLj^c>Hz1<*MvG2hiMJ7i2%qNci-iVnoE@9HM$< zDFT<_v_@!$rS18;YRLuoD*2dy=^D|=Mb+%s3>up9rd|BHt->iEwAo+S#+4L+!v4Iy zsIEBnRuSvSU+|$sEnwq#=_YK5*7%2$s9c$r*Vexb(^%L%qBHf;VG5*K3&f~geF0@_ zpZ-Gjh{5}b-0ah50`GtnkGIN46Isq1nPeX;f@d(mL@AWWQ1*b?)mKsOScFkC#2c)C z8g_zMUA5 zyqsQwc&uHNl<3b_p`!#C0z`LT4F8cE_Ai7GoFqvGX8L5hEqQ^~?VyZJq#aF3;e*~_ zYfCIcUPg3xii?9u%5wI~h=fDH;mI9-KISiaha$n+z z+x-`lEEGH^1MN!_-yT$?G6R#ET+g2jw_)!FwoV0@Lv8(PNnsWk4jwbL%+4*03Q(;3 zhdlLz>)Q&@v!{ZHSdg6C9sJ?yR&Jszhu7}}hqhJGQFE595NS-oNqb{8(E%d(ZqeOx)p%6$vs z!eDnp^8kq7eA?n{0-Z*>e#cJoE(^TlJn_|B&7Pa$gLzyTmX-$v>u(Vb3NgA$mlbak zWf=3Oq}f9JEM#|nxYWMkMrJG6_?c^hw($FLr4=i6pgdjO#_aSB>gTuZi<3&fq9&Bl zzx&LV;74@M(q<^wSLe>%Y#v`2og3^fiTkZ10~YkQY1t7Wp>xi!e<0Y_FP6*FQmUv9u9oEblF~TDC z$UufbH8evs!~QsI~trEzQg9uQ7Gy$;xu@PBpDW@={2yO<-7 z79t?8iahg^Ga>Ml_fVjmK?T)&xo_`w-5@{#t(QgZ0yrSRj6Q3eyO{-LWt@=Tw#|zf z=Umc4{muT&w8#3qi0xYOd#!DU43_=rxhK_YSo5?C+aRXwI_9I~zocYqc@QKYXE-g$ z2dk05D#6aXwYkzPG$7Z7kal~DpPu=00Id`1oPG)agx2QXsIXfjIc_p?YvlYm27;R% z$z{SEn&jRq@`$(5isY7@0&H(N{)i^@k9wxtQ-Il-mOQ2UoZVJ^u-ztO9cgJW(0D&I|AP(j!8iuB^Lr>Y7T__rAi^XbLl6b|MbsLgb88!Z-f>|49x-3A zrvfIoVPTez#Qs?B2o}$GSN-s>AGuF=;bG3I{Wd>@T+;oc;`<*B6(bwOdEGpDD~B}ZB!y$%)x1o?3RYLX9q#G+Dps^wvS?7)m! z*FQ8Hp^qnsfq8|M@R`#IA!sur!gSR`Rq>220|BnhTVF{-0)Owv!$q6v*bn?Mg&P8% zMyX)n5~`|%wM|?%?Rq{Opy9u1<$lFSIZAlnp(6ds*8%ZTh zs+~y+!<9K&l94|FXgR$p9w~0)2smyegmbSm7N_!ImMoenWEr`?)^$&4o_6Iimg9;X zCs0Ct#M-|$)Noo##vc}ILApQ1Hv4^i8&lXm!yoR`Ih7iwmiI@9`wC6}N^nq~&x^7g zZDbi)Ricgz0+^O@AeZ;&N!PW1+BkibNJX7_w2GN2o&H>O+3TAZn;2RH8 ze9AHLIG)fE{WobQr_nfF-RC2PIZdoi>gaf%wsN)awKN7&8XEQGWl!S_0K4Hstpf%) zSUml8C>8QV(QG;`#&Y&XJGr*(Z~C%l70PDk9sEDMtxQ=WPGQb2A0X2ipBCU*mWv(ynA@rgbJMYq{l_4( z4@b5TM|D24{MMt&2Q)sHEX+ffYDmfz%=*iE8{R2NfZ z!X-|PDwK{LEC;|A!A)86xAIJe$F=cg2Q3*cw1EaTo4x0=@}G!7L+0Caffmt>-_U}rw2L;$AE=mny0ztaJy3~oN_-g;X zttrd9KBa&b08qfnivj+Nv*=W_iR*G}2Qd3j4XY9t3mMHmhS~rA)X#AQu;^;w+`r@% zW>B0tp6?trMxx%PaQ&duLn~$bo#j%JHLfcqOmL?wH9ZlJMYh7a!l9^r&7Z61bj^+i zJ_y~Zd`1g$c|;m|{kM&a>!;FA)ii{(D3Txm&sn%S(8atC@vPpfFDXF1Nzo zo-Vqza`8gmaJhKjT3XLU%XNOPmPw~t0qT&^S5L0;Uk%A(xtT*`?0y+ls&=WIy)%<9ceV>Fcy~9&V`1{VyBSMUn5n-#kpJp+r7=y5YjxwWv;n zxmY@V%lQEGhjM+`3nRf&%qIf`FxTfu(?cFe2jUQc0@cFN7eF-YBjeq{SA!1jGUvU4 z+LvLn2%==-^bD?H`R60qpd8i=1NS3b4(`ZlQz1e-Bp|>9$zZC?yFNk!Gwc{;&kRz) zelLz(q*A)t$NCPQ5ul3=6xKdT5#%A@5O0Y0?eH61?vQgcfpuCD<&;JMbz@#{pPxH@ ze>4%qMD$fvU(t^^?z$`{l=+z59c}2ljq~L7BqtLZQ`)O*geX09ur9%U@&8%?%10El zi?OX?h1IXh%r*3s)+vQN9ZZA1vj~W4aqj2$rMRt%Me$bfB8xZ7cC=<_0BHJeSmr_( zN08CLM38$D4eUNL#nF)#k~E(W1E))I#>@O@Vb zY067G^1N+`4JvpL=8}9=dGKX@-|29dD7o`I0C!JiQBtG9&jS-2l?$I(C2v#d-{#Yn zDx?k9jcNe)!+X3y7>+*(4e=7_cBlr?eBQ(yjj{Lb$dGf=W`bAsiQ9TtL4%1p5Md7( z@W|YC<7*@6`Zdyt;mtLh!HxSVtB?}Knz4j0Rg^!l%Y*Is(nmugA^_@V5jTnWX=DhP zsMUxw`2py(S6168=?w5yJO}f!xT>Aro!who5^D)I!xM$>W{{qkpai3Z@kOEt?%%=s zR&x}ix$8OE7+NhNP;p!W#5~f#0rf4*Pd9Y1XLOF)xe?X<%S#Ip>@CUbQ6%}v$;tDc zMc{V4PfJbd-oeV{J72>ERPkcHU3IS17|F%Hr=^zL5IF3HR+DJgh;Y?WtYYW^O_^l- z`Z3T)td?5v&oWaOJt+S6c#G7W#~YkwX%U5Bpb(ZDH;SpiHQz?eyEFbkBelPBLvnI1 zeM>r_`rqqFJ-_DLd`T?&cS6TPY`8E9x30L0w0%mpQf_}=d>PFqc|li=H!U2q$P>xI zMS-T7He-M2!-K-|^s1x;%WqJ}gn>x{7VX;Kf4|a5+9yo4jZUUw3yyd@5rMsE*z5P` zwvi~H61}eyhZ}1&1ql?3*V7H6?tY5Xw+u#|J;m7<_GxnzQeSWs|9yUb6UFB=h2J!0 zFLadpXYQ4~rq4eCq7^sGOLXu^1L_fJ3`Z1+Xn>{RIrCpb1roIibMb-m*Z!|`3PYxT z6cT3}?)=poYf5Ahw|*Q8Y@MFOau z7LlmDkDn%{mRtI4^R^OAnITJeGxiqoA7R0b-b+~Um!kT>q6?FF>hrdEx=!$Y?t1E= z)1zZ0>lbt{}fD6aEV~$aUk%ZG1|)E*~=P*!^A|&{F=;G}*K^{tB$u zq^?&LY%5X0%0+uOmjP&prRu})tIF5-$%~Xw#H`Rj_&h|HXTnQ9|NMF&!0!1DrZV}) zTru6MW&Zb%-4@C`bq8M@0kjFoD^v`lx|>A0*7WRz6RMlr{2Og4<~?i}A})#ml8~e2 zuUk}rp|BTWN=xyNhV2n)=Gz@ps>|2Tp23PTJSRQg~0y%brYm7^O( zBo#M*K{za1;(ITdZkFpQf?3^Vxq?>Z?G>f`FUwIB^+`qFI@sKGyn9IT;o`E=wd^h+ zarB3xGU62=198tgZAw9Kmg|j9g2R3$RIWNJrNeiBcx#DMZMWjh+9(xnSGA}$Dg@Q> zadF&N-{_&bs3Z=}?BBuhA{S6Y2)x_vcteE?qt80M0s}!rFZhHQYi`aS5&ktoBoZdB$dn@2vp%?G--z2U>BGBDM4k;1Mx`c^q56yeX)mJ*$i z1UX?QGj8m=WR&(@1S7(^~9->p+5Y@2d{ zA?iI5r!Xz$pe7dDLP5{|Y7g1_(S*0?fS@YT^>>jZ(wC~*JjsHgT_>L-%4Zx%?1#vHAV|hnL@+mV;hRbayP5lHeISaUi{1XDVKw zq@H8tqR0Plo7Ol<;{Qls*imm!rKP?X6RD!y+;}fMKtvkU?`FaNa`Kq*7#(yH+>vRn z-O>Js3TyatiN2IoDDKmI>s0J^iIkvNkI^UYE!Br}^Z|5o`^sJ<=jE^^HBrD{$aOY} zpF(uPbY*wpA%n?0w^vDu6EUZJe-jq(&vC_rmi&@(g3Fz#I(eysc12>jgELC;;~AjQ z?O}x8mDk@s{?+!w;6MYtlE}XI60K6#KerX$^uv#cU8KAwFbju7xy4!h zi!n!uupa8%b*el6W7><)IhUv&&Sdv_dXn`L{?ypn@e+5zxhd9kL-S64`JGUzGnXR>lswY;6B9#- z{bz-b^H|I4DIT@+!7?7EGY@W$^D%*7?@1Hh14>d&&BTsUPWQaF-|dfvPkv`mwDM*( zqVE0zp&FkGy%TBB55IQf?IBcN=LU5b9h0f997B z`b|1KT+ftZ%?!8qvH2dnD9AS4QCx3nGHHN^tve@Z&AmndhK)L3%5KN=Mr{8q4dY)a z{_S{smg^siSu2f9y#>kpnj4du72RP&+}YO(V(K>bgow9m!Y(Y z<|rm!*J0&cwuT3Wq!(XmTz*%-u$x(>-TCnd0nsr_sntXsFS4`3tROQzu@Mhl&yTvg zqdn&DDP^Vpkl)_z`WxkbRG8{&e-=gnT+iLqO)2roUv?zWYL^>TtM%mygFG7s!)!Zn zGoQ8QC6>q8@dmbN#n}htQ!NOiQ&QL zVBLa{`l|ZscP=8xLnl{=(!J{n<_*s6X@U>-uWck5>iKX%MX3_aS4PDBfKM#!`nkRW z3BGY7Jqx_g|6vTyL8yP;)4?4SrU<0(=yyK&Z$&Z1X00HB97Vwt7=|s|vyHpD(M{|A zc7y*VFC-V^OGUD+llM1VIJscE-qjk3bZs;ezWne5SE-~KcgppHE8Y+EUV-Dr!`)d# zyxKSSgjBm|!1>_tmYC9fhJ(_y6nQk0h#5VWDFzYB`rRc7Ymy1Y=p6O-moq4Jgf zzV&O0q&pk3Tl7+L=kr`CtMd$%Kp``%mm`e)!`oH<|_Vp{=5{`(XGF>CbA+o zv(Dpw3#%lgReCNS7uv_|hD8CG@XWESzw(CCo%KD&vSV}TTMFtyNBij;8y@2QQKeKPGnaI)AGVs*2ZE-!#uH>S^wE6fP3*Z?HG*ete zvtQ|?Bk2gGC%K8}?K(G@6=Y}B)ubey*r$ zoq8smdT&dfvZ`vw>HiW&Uu=?K*2%Ukqfa0w?lU7;tCMZI-88pNK1M=9LXAnQgFx*~ z)+$gtmtvnRj_F>F|0k&jgEg4%Jdq{&BREtyC0c)PtVC#D6(GlDjDoEC*WHcQTT2x< zMa;Ejo;qF}M%Wr4BZ1RB#oKOcWA<r9PLA=q~W#m z#(DZVpshpVy_xhDYWCCn_WQq?%3(=Fj#!EIELY>-d*51}N_nVrRlib!Tp9-~2;3k; z_M27h52^i)7e`V6#pV4XA*P0C4OXgzsD{JvS3J-K*^V@Q-0V+3Sx(coGSt*gTbY~? z{VYV%jzGS)3Wu-sAYjO!io z=1Hgv`r+(@2uLbzUX!78=N6M0eIB;9gkUmJNiF9ht89oOu~(O!UNgZtH>dmW%H4?T zI~2Mp&NRKsUR%&v$L3`O;D;joqqyF*rsi+mkF3Y8N8a=<|IRz19765Idc9}KvvIN5 zc@DD}wgzQ3<1&VJ+TU+D=`Akhrxgd2AWnIB`ntdM@Zo6<=i~)!hNZVu|C1CI(|(KE zFdumx!x@{E#QR4Bi*qWN$~uTy>6VCV=|#wkPS_`vo;iPGP2>&Z^Y`mUv7PYeh#P1> zSv0Ov@=TVU7VUyB@EZl0AQcq-sAS8dP)vjL{E3SA>xUpmpMcY5#Wggn5)r#d7)_%3)y2_N=1x9Y zcY@Jgq4Rd~RYb5KXXIspn|&&F`cJ)bE5_HT>)l!M9>!iTSd>AcW%-BE{1|b@nu2rq zf8m=P)k(Rjhbp6xg^vkpMV1Z`+V}(nD|&_}#2hf<+;PaR<+CN?=(`VWY8lVupD#sR z=l#M<0-Zw_TF<2^;b1whY z*UGE(QQDQr)|`)gbO%)?M&#GryyLK8x`B9;m%P#ro7cd-b@;$t48eLeUJ1aafbEUl6rHV3+r4RHWLFw;Y88FtA`%e$ zDSponAC_a}Y1}B~9|Ig&UQptFw>oM;0omdtT!wuT!)vXoE4gE>-L6$cUk!(U|M5=6 z06g(ro|pN~H3VO+E$#RjX?-Lz?xDBeloS&^bcKvkBiCL6p0X~%6nJuN8QZSfe8T|c z+S$vq(c?1uzfKqk!p!8ogS{8X5l#Pc>;zf<>&Q8$_nYGMu{~#%u2N zHE<*n&m0BwM-8n7@K9}wWx1S_6*q6CTmIhJbMMF13hxl_RD4?0M)NY=Uq)r#z$h2e@;cW*IP3gIKYH%Qr8ez>v~9zc z-GO%gYCPiswYU6!u3e{}28QGKGdw$KJp6fg0?~woV2){T;?v-!1~NE0Njq&wvT4K+ z^!bLa zTZZrCj`_s00*x-g3aoPifX)QLX&?<;`YC-4(j-}d4tBxn#sv2u%>o0tj3mQ}=Y`vh z+Quv7@3cNPMrUEFd7{_?`(#d=5H;Du1F`)uZ=UrbiFbn(IWLFj4i)}UE()-2;}6*I z+$5_BCR8NxL)5O56;Czkoj8QDJxV}J_A0JTjjSP}V8w#CHfaReq~SF$5(Uog|8${$!Y+Rf{0rHJ zmYU$7zqyLqfAPaw|DSVP+@3kyWk%^s#Jx2zNL=!ClNOcN@wpf0ZyF2R(`ySgte+i| zudl7=6wi_@XjTeaTv=wn%tfkFI1#&{2E zP=-;B_Hih9JT9|iXvC{vM4;sS!4o_T&%f6rYZ|+LXL`6Dn-f3uz>zjsov|k`x9~bj z@`!TUJLA4Tz`y|j;_p2EX$$r{=4gsAe{k|Z>WB2R?MfKaKAoF0-Z6PT=ej7iG+cD(IX2t)^=JN7e z?B4~mbE3+CIlKzy^EHfr&x=R8_93=@{LW|hV%(A(|6Rfg*Z*a~rp6^a4Qycr_Q**s zi!WZO;J_3$%rl!<`hSTAAQ^E5W0_-Xum6n<>k8y?{jhp}q&@iaV+!w2bKOghJ^FIK z&BexA)g;IB-8YVRB$ny^d18gz z$N-ZwRhU~x+TyP=>`U0ya2c{Hm1otx;Cdh&Wpp5_fey!P=!0=Y7nuI67>L7q*u^MwtvS~r@ZJ&*B z{M>{a{Mj?sz}=tJ`y%{DuPS)&X ziwf=I`w0IvPSl^-x_5(G(;$E^!5;5m;wk*RU{1nU?&;uT!@%XdPKv|>Z$0k^wskf= zdr^w1YmwX=m}o%5jJ+@8=Q( zkSyUT!7`W=BFF%AGCIW}%2pi!_R7VZzjVzTj|%9PZkN578YqEoD_a0>_TE9^VcV$e_%WoRWSX?h;i+tzE-K5a)H6lh2C2b>(eJHdsYO3 zN+JcH?ya4*f#E;Wu}WwFW}3M}>c%}fen1A5jihQz z>HbraC5LGmUxOCM-b~v~-X8P522iu2UdD41Z&42O@bjum7WGOY%4-c2z)?)MuIZv9 zTt|3z-*N8iHCEIA!aqR&7bEH#PKPH1JloAH*%3}2EjJ#{VOL;%uxdKLXK?bIE>dF^ zi{_!{vP*hzYF8K<7u}7|m_!WVO!^kYgt+ro5y9aEAnAPlkE?7p4SvA@P`;mh8t&jX zHS7ABh?*}y{f=3R^Mi7A`4?`VXZKhYF8$U^IhV{YEutpcIPIqf1RJFMF{)#4l`%d9 zFhgR{zR07a+5FA?+_q!rN!{bQTX~Djdwa>LrDyA}THlt^#1J}uCO>nO`tF6&XMbHg z-7lSS&QkU5?}>6Cp1;iS_EsJI3m!&iRZ?}Xe?ma#wdX2FgO#c{+J(tyQ`dux#76Dws3{-4;A z*`P1SBJ}i0(a}ActRSrr=NpkSsC9)ltJ)EdVWEtTTvD+%3Mx$Nw?8oW;otxyCnczULosJg{ z$TdfCt+i8sXn*p^c_DFaP-fMN^UXmoFje+RnZt_iCp^W6C=_+lLj>S$v9GVquH>x^ zoOm4fa9!V{F0#Gb(2Zk6z>FqZj-n0wq4q|QXUnkdPK!|qZ{aWe1{_l*H|YSK$)t08 z-A^;e?~&}u{*Aax?c{iVpMnsAi%cXPu5C-y1P(Xj=fIIN#)9mTphi;3*U49MWHeNm45V3c69NHPnU7uB;Tnq-~j8bOc3#BS$HvE zeY&>t$?Ff~a=Gb+23chY;om4sUPZhnG8Zl*4~17r9U+6ZzSKr%EDCOV{RmZpeleEG z5qeiw-4Ff`;f2~5N>6xo`57fjCvGMpB@g2NuLWrE?)tD&{KjyP$-y=NiM=?F%zNVY z@OaGYgu`{ZK?F-OQL@k3^mkz)^w1y>rrS@Js=GQHg4)TnFW`eg46VmpV~W(!`6Zl) z#-j@TP1NxSiwZ`Cg?&I?ungE~4n%7mX8P~l?kBw58k*7S*x5)^MB)b%bccnQ;>+Ba ziXl$pCw{Nn(SZ_EmP+|l{WneGSJuCUu0DDBeQIy;G^i*7JEhMjMidf!JFys?UMsY| z7;WqF8!I#+-9>-GzNt^xyu{M^6=PNz*xV93(ZlGC^}h3EFl#}ST3=jv_k|%v30%jq zzTl(yQE$}RxdOrC86T8=q?ILD_aBVvLJP7SyWZDoKTWr@k>hLEnBiUci#=SRhPbII z2pRTdV9zg6|JXyBNtKAld#8=h&7s-H`v^04+TG*lbeojp)}yhjb=Wcjykd-8D2nX8{mi&ODi6zrB@zyO{8VkiI|Iu`oVNrf>*S=@yMx?vDLqHG)X+*laLApDJ{z*u84IxN(*C5>} zN{Dnf(#<>n=Q!R^^ZBmXYwv5Vb3HVLSB1Q0KSRtiZaLnu6JlQDi9K|0%)=f()IAon zoxPg*Zk0mh6Nf|Vam#SDrL2jk3Y7-VwwIg$0L$N3K+qSM8CiDbGCD!>f9o;wtZLM3 zD5|!=IovdRY@I|pF46)xM~cxTV?+$-S<{*A#X3@CY_)t!-K5{aYt=(}NU8FYegB@~ zmtp5<84!LgO#(pYZ^+GUwtq84E%8kCe$wFRmcGp@j20pOK=AUV@2hI$v~wUE;}d^I zEC$@_P!r#l$NXhw&JB-UtV;$2?NwY#Oa?m90A+u=hD|A%Tog;A^RZE|!;CbnzhXU< zD6h?1-$XbBP^8V`0-4iIFSst8Zx2FOUv^ZBGRDxI`Puc`Lea0oIejlou7EXG1hP|! zAdQLm1HA5w3IGYfnp&;;Mg*);)V(cr-^NXPa_ukvq@r$o#@&V$JbvH4HNnKYx@c)z z;(J>{&S%-_qu^*`Q~~T#jNH>Gg8p0g__j8$*UOITf+p6B#t#6<>{<^@kLODqXk8%} zNRPaS7j!xL4Z@h$-+o7bDU1sFI40@~TH`9Ovb6@&K}EK(IZS=^@;7&iNp8E0k)Zbv z$ikK6J!os!)7?sm0mk!KkbovN?}Y?0Lg4Iz=P_~Te=G9yC%d}3BNRD&cu8jSXhfWG zr>V{>3XU+>BJf70+}BrKLSYi?V1V7L@+*_2{Qez|KAO}F>xdPBWa0Fps_#Cs&_+(x;PZ&C zE3Rn>6IO_W+n$GN7J;}=;-M6r7BOC!K*uqD;c364v0@SkOKwaZdZOzG!Lkx{YI~l< zu@M`I?u-~djI#{@kfLU+9ri$hwRICg-8tzNuEx*HegK%L0uRXa+Pgyb2SF&Y<{8Nwq6${@-OpX1 zIlTxMmyI;Af?!@Rx2vvZRp!!xLvHC6Fx2SYXo0*azP~c8_!4JZB9&U4(T*C zwwrimrmm4?9dR2M!((hW$}@G86!yRQw)9adT-X32``t5Uf(fc_YtbGCv4+2}unl z#iX1#W2x65*{A0r7SKo;LxPyuy8~}O`v`|f-E^}_lAXt`ctYMlQjD!^nbRE>l(YJf ze%)e>cjKar{Ur&AVe7W*B6TLxd~lw+9^RNd6$_DnF=|=u#d*NnB$sMXLXRonM?wiC zu(eo4jcXY65)C?)wtu=n7)7#jzUs4M<1Co&WIb%J_;@(_lLgssfmKd)C0Zp`CqcK` zp|6P#fjhiwL?Ra%epBf~OA4HM@Z5?qfzSPcNA5aQlF!5Ryi}!_jSK`m!L?QWPTKE@ z*;$4-jT+4!vgl9SQGq=}bE#K!2~ve6D%Ms{Qi7@GY#0+Hn;LIorvc#QLC64D20`Z* z_jm0>n`ss?8&5XPG$JZFv9i)=KG+MC;iS6k$iuO4L1X>sv|r>OL=RP=O;seW*GU-qv#LFF zDY+-_U4pQ2ff)q{JjDf65JJ;|ngLsJDN|@I)c{0!qz+FQ1X*Lvf z)=Du=!aXS!p@-UslPi)cTy7^uL};WRxEG`JMvgMK~NrX6>XO zn;{HjYCZrQYhhu_ieSxg^Ld!@Zb2W#S%!jHxsYP%%f2xu0L!DD>+OrjEyF@Ld878+ z#946n>%%YCtv}k?rt%V^&+Lc7h@$Xa%_qbu9f<@ncF5iGe28?guPJIee|A8cMY2Lk ziGoXH*efjLHefva#uG=22Y1`d&X z>**S6V1eyVPZ~kmzgkmE)9Wcy=2y6X4(PLsddGY9%;AmA+xnt-KLrN=F>7TelwU>3WGQnnL}R1Bu(1!k7l zaLFy9Ot(Bc!~Oh0Pg0|_WQjr*$bgQihu$V)#yGvp7TdL0M!efLY#oJ{sF8ft_}Y=& z?S*d}?LPo{ppJAj#K}+FF?Ohca3N99W8V)Lp&p9(Xz)B3WH>W8q4BKloxxWxYWu(L zU5i)L2wK9#a~^VZKVqF2l_`aJPB9PJN0522D+@(8|N6tyo~Mcj(F~Q15_c+&efM72 zvzl>KB)Q#U{Cq#5p9{BQ;fWEVgAVGhBbFadb z=@1O__W}ciP6D>f$aoOMHtAD)5|=}ja&G`hGqnBWktOV`yb-C^;}W`nPtRMS;D z?MrU@*$=;J(T_zbv?vrLzJ%Yp!MdDy!aK{8?&2>6PFioTHnZZwI4Bv(18oghIknl9 zLr0I#s|l9HqP?eJgw1z-+WReQN;kVEm*+do<~(3?vcbkUhNUj|n^$bb3%O+9IbYk` zt6#e}dIWx%=ND{c_FWe1zij1*pcqRgF;q=#~O6 zEjzE>kK&@%J4f;H7wREeOiWPIeSkpbNSa2dL&1;{xDdxbYeY41psTd+R|Z6{IqbT# zYBixT-La}`ucB0tb6-?PD0y0^?)m_d!qjlq2VOr_7@71HQQHgOBHs=zk z)QubAc$HOY)-Yu6-GET*2|C*31Q2mB*fo;v&v;{{A;*mkz4yszeG6?qa@~M<*SwwM z>C#}o=b1XwbKio1s6c&Y+Mk4(ODno!xA!SHfBUSt+Y;IUIF)L1u$^o;VgNtf)_+`3 zlb@Ura;s})k!~0|z-MJ&95A8t_pL5MhXohWcoA|e2xG|l{XLfs3DJ|d+jl9X*kc6v z$aiT!w@6AnU||4(uK+X8H2@0xuRWM?vSP(kt^wN$$Iw@%eBW}8Nq;yPk6DYL3w4NiTj z8$Nj7`^*Fal#{VHpBab$Gip?=SWBILPn}fJko4&jFEwE6{>jVFjLUMRxbA+F0)>rX zT>O!hfAUqW#bZn-smNCq43kH!Oc}0gyBRMm6wSK3M`1ZQMeuKGwuw?EtX8J+^h^qM z)1Q{@kI`N;6*JtXkrh=hx9Z!^G_vibZX_mM)PTdl0YwTV@p~}=3X(~C7vp9*k>Bo8 zh-^(eLFa&$E_u_zrED1qtl)Luz*W9x(ccP3V(DaTrCOuSJK+taSQp-45^gc@G~sl*+ky&QO&dl9NPdp{h5l;F z-=tI;<8zJk$btE<(|%tiUYX{k-x0lnizfOsO6qTE0oVGImlfNgce4PrHt?U29mqKn zb@@uLgc9nH2S+=`hc02kmPj@`KGUX8OeRV#PLs2PZCu0|nbgL)GcL$EDkV`7$xVVR zTpBOhL?>*53?&xiNKM}@!5KvzIocyTzlO_FT@v%wO@YhlP1y0SZGe0VN3r5-FMabckE2PZlrB@(DL}394Gi7)YS)OTzkmnPH{+K=I5AlmZc7zRR zUBvx()~O=sZ~ZmtMZ&-of*k7^E57}9N`+ zwrih@Mk?%t^KUm>0{Cl!$@a-5$uyLX(X1_Tp07*$QGFk@!#JRG>OSs0jU&2TS*Ms({ z5z0}8&7r^H&hG2RjmbT@gnca= zG%kHs{eBtSH ziyMH1=JA*@Fd(Duv=Dyh{&!pSI9kx77JSa40XAO0_PJqy(205gR-`l=7TWQ^k(8XP z51{1dl}Azi@z7NoXCs`4x_j_gypjJCWp>lfI7or{%cyQIG#8B*WKRUiF~oikG84lB z!3%mXyv4D;lhf@0dJhXl{LU9qaTG10MXvEAbR2risp1hDyGUlBUpuyQ5+!_Yh15pQ zN1f{I{R}8h1rStwqO5y!Y(4Dt8H_kh3n8?amDyxb(FB!1*#2Nq;a^W*j6Es-p|Gc- zz>8R+=!34wG+}Q`Nn{Yv2?{3qUk317n$$kJb^yv!fwu=zRhli!J~-Y_$2Db8#0rs} zjCIZ#s;esIOgy$8#Y>EyUT(^0Ua>j@;K~(W*Ep?pt(LLBFg{DASC3jK?1Vff4y4a9 zehx|bk{+G=ljE5V5jZvB&q4c2k5iT`rm(qSCUXQAYhlpy|t+$uvYFji53waF% zG2FfpV@K_qV5HZvP7S`r&laJh!&#neLOvk?fNbQUAx#L(V}dKlJS7;dbu}KK`)~9JA==Ck6!5+RNsBMX}nWl13DISJ`ZO zhEh||aPL&Baq+D;Nd?=3*h4WIQ+=GHFFF}B+&B(^X^(tULmd{?Ux&Al;uNGAI{PJe z9i&T#U1CA00HltmcINiN!|S5Ho2Tj!$qus+q7H)ng}052=O1dH>#B$VIg6BgvC4U* z_P13J)_G^ibuTfOEu9Zs|XNB#{i{X%bGLP7H+r%9m4c_3)`J zoZU9v8e`yHb2BG_N4*YyKa+e&m?Aqb@q{_)7oo1N{J+mBD<+0~N)ygtM#uC2?Rgy= zAMF|=+$$B+^6{co>euP>9T#~i_E{Cb%eA7fj4lvj*9d>8c=)<+Ok((>eRvq*H)0J1 zTtcft-2`rh=U@6{DCoT0JlouL#{Mdzsj7VQhUm(u6}qA1Gy<@wZIkQO?yUQj$iv)K zu-%z~35Svt3HMqZ?|1Ugj(N#@`CNAUVx{6Q6eI6#x!%6!ZASe|jdW<_!0^x*9I^^c)nn;6*3AWhi~H@4_w2tZ-SG20p*?uArlkVk=7ZQWXbwl!eEvBha-nA zA5)*^62QOwrn#9d1wqBHn~(G&8h3A!FCJc>=LTnonEB1^nb%{w%+1RNZ_hu=sc@+( zuu8;UK}|RCst75Gx*C9yNJM? z+Q-IC7U#r90mbb@VZYbA1c0w}xm-Rz%(q?VwyP;<&)o8}3Hexe`l!^*5 z&*Wo;9->FG5jd}sV;M+FqI@{%A0&xwdw=`i!Tp3F)4u*VkTXr9o3=!nj53sHAe#d1 z#Dz@mSQuID4{G5I7aml|bg0Nv1|VW36qhx`puC7IPmPl#@@gD$rQH>$FtF1~jHh+{ z>PVWwQw$GlMXu3e)k@7kb3|Ip;|mCw{e;Ww!Yi9 z=Q`DlrAg4aZoRkyrQQd|^%eSD1gmS5d?&XNPcljWe)C0vK$KKy=()Ut7g!M$f>!>1 zMEm`qY33lq*4KXepyY0+iI_ymio1NQFa&s!ePD63w6UPX3QCMnIbBPJCX=xHok|65Byg-L`(#sDB~`(ZG$q z^SH(y;orx9zr`2SRh8_ljbkRZhP9aJXu2i}PDVOD{?*D~46!4Exeb^?8k>V}PF-~- z)@uGtKh%>z=n?(7jF44A9sk$cK-S2yF&h^{c^m1gHJ#5;D04ytoZiT`ihK=* zLjedX*W0Gpv#Sk+cz=^?4jMvtZA@Oj*IpSw(QG^gkV+T zmM;;&(KSNQs!;L5x;H7uQ%(#F&X3ULHhQ$)zkXomUMe3AM%FzV+e60=L#4z88bDCf ze>u}P9I)Jn1?>2yGieIZMKH-=*6A`waTz{|yp9|Cq0Hh?;CFt7zRSkGs;k8Z8$=F3 z_gkGbMY+>x0Lb($iU5(3p3YWB~+y#$A@X_!AE8@r@ zV6`8PR(hrnsW$+1eYZXTHv7D-8>}!@HKou4-H|vF*%F;t5$vV!H|}Jx3(mb`9Dy$x z0MO4L&EAQZ0cO(A8!z3H*FYUvqT6gM8W2ohU_;$a*^ygAJJ51GI$uPG?+$JoB0DM3 zO@W<>BTAE3F&6m#P*W+wWodf_-&mbY6d~fKdB!PhBH~%6TSq>Qw>zIRu466{j%5FT z<5>P)OH@)1Jm~P7o`C7etf#I55&$^^biY!j3qahU8m{=E3)(=CFF=k`#o^dCLi%7T z9Sy%MT-I0-ir*1%JEmHz(A2?etN&`FIaeFzUbASGlKlBg&A#9p4`gK0!!i;kkIRxB zOnk~X_!w4SnSBFyTJPWrtyl9+KAP~c`ed!$zqdJflJ~-KKZmmWod=4;Ql0%DCGb2| zBTZF?ze3^e_7AynM+-%B68g`i&gLTU_Ek}2z^f2hoGbM8xc&x*KE}I1CF2)G9dr0w zr{7V6|M$$u=WTu7_hIIlxz>BL+vQB$SnnGVebv0QVI&}1>}FsHx#%&3-wr!0ReQUB zaZBoWf-{VFO2zYP;#JOsvL_Zh8;J_<$n%<;1^P4Jh5~ zQ6k0|Ate`<*9g#=^>aT8NJ4cB5xQCN&sV)f+ThM=qUWf zSgucv`ZHUv@f4lGqn{Kq`Nc1c{4Z&54Qc#s%`y>2XicxL<|(gb8)MBVJwI)uSXEEg zDxmxY(E-T)k=4rKRl7qz;~(JIqzPc%lN)~-M<>Dg6$eb<8}DAvFiA?|H}PASiPq_Gp5R@JsXYp;H&JJ z5LfDh*YW>?oI6175W0QDC@`E9mT)rH!WMt~|y}#oVoc!Cti}OAygSNEXDJOBX@@9NYB=Z0sSLE4@o=(rx z6^wM(a1gKvSyuigjFnAH^BQ~=yGL%CwP`!^&tAP@&wua?jxKn?+N2TI4z8Qp|DjfC zn$#HCl!r{^k1ame&aV8ZBFt!c`%P$y%g636#(DS!FrEuV;0SAaA0 zfSN~h>1cD0tzey{#oxod`TF8~RJDtoPn*>*E<6X@CasG!%w9mdd+;8?tq{Cchkeel z97o$}?)g{!!ufLD-^eNlF*Id`ut@!AEkJ~E4=Ca{lMp+|flmmYlI(~p9yZbg#b5v7 zj>q4f)r{~4*iEF9p51%<92VXuc1q}$ZmQnar3G>AN;zPLlv0fv#}+`z3-DgNZo!G= z_eeZbcH3f+#T=;)4SgTm>q{75BlrKk0N&cfG~(=^E+sxo?a%UlHa{}n9PnpwI@xMg zCx=3wp0Q&xouYsP63xWGPb{lbIcRW1R7?D&mJaTNyMVN|msz zo$4Do{MzqI?@ij8)w?TC#1V4<#_TFTa47NuvgH}}fZ1JI`utZgb?DFYY<}BUNt$<` zb|l{v2rYWzr92Oey*9gU-rldPIuB#}$;{vC*M<>BU&mSWx|kr3EMw$0GeRy!Lybi8 zq%vv6fAAE**>t$Zd1y!UO;{5?QDcgx-kJvvN!>O+hodnfrb_DmN=i^U%7Ql<+wW0< zK*=&wu`2&7ZEYQE!Di0ZmHWy+a~TWD)?U)Ns};p_e&>(90Zt3zhR{!JMoRw~jV`@B z8lFeGKpWq6eC!7~A6RTIPoV$`gIoZ-UbFy!D!o-GY(@gag(IG1cY1%~Je0K9;!=Mk z<%+{Z#Q%Ox#|!n9aM!}W7DNo2Q*wOzS2zq`Ks#mJnnCVzVDKfK)kT_A`HNBZPc}{= zy%&*#dTKd5i$na{M3)nUSu3;(hfL9j+C2Ul|0*b48u;h0GIRM9nM;7>zYFHs=Kl7* zW6Kt6hXVtlWW~xp?`9?jH;&Hq4Vuy@11M)s=)U$S!*YjOGG_2J0eK5k&H8Hkc`K`M zv$A}eq{MWT4C_x^Y5pN~4&pDx?WlxyoM}2G`SMiU#xMUE>G$#TDMm_L3F0}=%%BnM z%qBUK5Qxx4GJ@CToAQ{A%S%v1zjj@~A#Zhe(ytM;S27`Y<~(5x4alB!`9rl!0r2Wi zF#6dug)=OOjv#xbgK_(FULhi_p64ZjC$qJ`Yl0aBo3#Js2sdq<`!5K+Ox>*Llm)yy zWUIn|O^ovH16=QX_f(X#;NkJ+-vggG+Fa7xLe3kWg4Oc3_2&P*@I8lI_aLb>R4Bb@ z)g17?^NV6sQd7sKjj0i>v(f#rEVO=7QO911j1&VRM8Ku_3PXI-XxaT3S;>(3XlY3y zqkfs@8oOgxFFgUP)&5uPT@}G;9S;)0gPuO!uB+|qfj1GSzmWg=t9&~w;)BY+D<_6A z(kF~+XLRFzvgt+x2I(^aDl8}%02#@=-C?Ic%bM|V z6}`T2y%=m1)yD-ls{209t$Gz~wSM{FC^zeOVP=!4)Nh$(;?H3GL)*XGHRiGH{#Q=a z@syUv%_Ujc99WGD=7$TYcyIz9wJ2*WP$?#80-dAHYeT6=y{y6|5;_2p{-AI{ot_8F#csxR7hOS|x& zSklDuLKbZ2`yS(mHD)VjVJsR?t*_KVVqu&!_I;Rj@WV*M*B#%^yTlIwV5`QgaFC@z zYD(lu`aJbms25rDahxsWzyc4LVbEKhcd>S2W$c0ZDV9EHL)Z3gSJ_r2H+ABM({OvE zt^+(yX1L6E5}fse3?Fj=Xk|4P6~K$`=|RB9%@fk%NF7e(VW)t?oq>@^dQ&`xnb z?mN$YMKy<-KJ5K)`&(jk(G}79ND43bon!Hz?r%fF2m?H1{g6y5Ujivf@x|${ zvQ59m8y=fR*gZ|?#7Tc1=7IKeHMS-eP{C4B@J!)K0{?VYP)H1&h)08+WN*J&PjSc< z(?iRu_==5`%iKJ0;gdvQkE*PRK|hx6w}A<`R%>~+ebuhbf8AXpGt3Dg;v|H5ZC;vdF_UU`a?r&7&>hp~1OKnNtd^>HPur?j#SRFdIb z-;@?}Yvf90(9?{CCeqps-R9ZcYSIcmE_pTcTI0QT49CEe4A9)XMr~>R-(@HM6F@>2#MV zF^g4jpT>Wakj;#l`x`5PS9~9R!vB0x4qT)?uV13tyBBncw8c*_IrmWz1wz zOdcwp7#MksVeY&{RJc-=ouUGKI0pSAS7;)jvfL7S!>w`oUft_OtkjAxi>m_~&CO?1 zZh>i`P=So;?mczU?Iw=(7t=3P)D&9X;||xPQ={z0Ypr4TobVOD_Cx^K9;olMCj8`G zyJYwVN3-$ppGFI?W{b#s%(rs^_PRBk0(CTI&);t8kETW^r#*XkMHKS8Za{3#?*%0x zaD0mdf~LAx|FqpQeeY|DfO$vAZFLzjFyS_LZ`y*mUj}+qsYZPJNOwP1;)b2_yk0K03$3XoM()E>ryEacdwJpG?bmP(*CW! zP+2j!8keYJ@L+mjkXv*wh-yuIakobXBa(vb{;E9R{OhBH0BbcrLVDJ6%6v%mrPKKx z!zxd$uc8(NE#ane>SzxIFbmjQLhDI(aLG=xR289!q1iGRk;h7jp9v|L^__T_EJHAu zNx6IV_RQN)&XKfSt!lY|JMjh^d?c7L^(4ougDHh3zpQ(D3DXiPcGCu^Q>dD?fB@~O+5Nec&nx&im#Q<0%RcF6I zF!t8kV+t&XtqJ^p%vw`~fkrRO@r=RCl-7u{rLUiNbXO}O8Yw1m|BybAUEGZRYTaHz zvnJ(k;+uXigfQ>Z5T1X0GVca=*&cQ79dC>-%da^4WK3rFDmtB&`U|C zHA!&EN*@(FNn!d{h>!M4Q8_xllUSN6sOIE&AXLy^q0bg{IEt(j3xI&^*dkW-qvy?f zH@^q5nKgOaMZPUySk)ovFvHsSeo_8~k$T88O4L1_Z=cZ`y?-=N>?6CpHwm4AAjg) z8+DdsKJUsSI`(cyg>>)XMI$T!e%t4vZx{Xi%-ayI2W5#f4t{n^wUc@4M*RD>$WG)t z)>!)k`{XYWKonc!Mxq)ifGqKS|ct2{Asc;g=|ItR~`Yik(z3O}0u8ZzfZLttG`ZYYkUS_=?hK7a5JGYZ8l337&%)MY;C^|q( z9;{c>{V17ge;kPgGX$W@WBSOf^4|Ik?eAK2zfMXGZz4bM-&|)u---2G6L9v&V`nkG z;k}z&vN}XcQ1Ybsufax2GeX5WH^qYf3P^QVs%TWm=ET7Q4iX%(fNUM4BWZkCWr3%E zOg?K&dM;K(|FOOIe~Scm&WR+Z5OgaWMJ0=Xqd0aS>1269eGER3zGG-uI<|9wx{(Zt zpb%Y3ClC`t`jnXs-dsr8EK=++y((i504nKxIAiW+oOrxOTUP=2 zky$Knd(i}z`-AQFSDqDhiHF6E5#E+3BQwFg+D1e=i_M2P6tDJ&tZRpx>uv%smfy`o z2xp^!^n&`&qiI%8iN-$|e73p}2%z?~!Z-wGVB`dlfj=S3IHOvtDf$48F%V1)v??yA z5GBcq_Y=+)uG=1S?k~@cL=*+i@yKL56Q!NTISk@JdmBZj@q-H+ctD&9l9zRaENa~3 zi>k=T*hSi3vqZH;Y=5a{t1LimQs#zTx)$HlsT!Dm(5Ia2fE7PUM)%;}24Vp_RDf$( zwFnw-z#5;9Xmv>sqv0BJ9WwWlrS9h%V`fdgQ~?;ADf9~%dyJ%-1vW49Cc4m;$8EGe zpLfKQG?L5%Lr~qSKf6}=lJC+q()6Jb5r@>S#D z`dI0eD)?MVY$%(ZGD!NDd3wb%EAZi$FnbZrb6!AG!+gPwYwk!M7$C}dFKhOg!~e8- zQR?%gH)Jah7lnc)6*^NBaqla&zIhr5>&jBNf@++bEkTL$TlXB#jb#A51T=>XYC!@h z+jD%z-v2d3&_qtIu(N{4+_U~;c!*R}Yv;dO_-uLn6UC>gpg^nJ@HB+5JTS)!S?q}s z>Kq{B!v4siy{+y@u>SF^`BQYB8VkrR0W6+O0s`J-HFjMFvjb{*shGV5~~LyGN+% zA|BLyeq&g3K=f{3OSe4oZILk{A2zqoLApMi0S48w4(d0wE-M?_4hgyCc^i@^#y^|r zKNs8Z>F+5HMQ^;uKcY_fgdM@Pkm~sIv)g*SqGZB?i@hAM zab8Zs2wxW2Rj{J~oAdF^g*UM@k+m-a$hiy^`RMu-qnP9hZSg!OJCFMsDaTG!7!lt! zVXs$db)OCxFaapi&$QGA2m(m%k|W!GL;y#Q<)%t!q1Pu18k%K%Nli2>cZ z4Q%+N*oZRU+kBiS)hK}xlv++)8t_F(V9Ksxd%7Q{%~c{imVF%SKBxc9TvdSQ<9whN zQfs9BppnnteBE#STeD-44=O6Lk2mr>snOJ^g&J&;%cWCwY6)JPW~h)x%3}y6H-MiY z!8~tzM>l9$%fQs(D+mHE3n-LMHyY=?q~3bzMXs_bUcX2n1NCRG-LJN3v;o>MA>r?N z(Qk=G06VMiyYwGhA=$`|RdKEM$}Ot_s*9vHQiYjd!1=2+Mpzh*6}zIxOc~8^5t$TI zZ_;9}ef{N&)+x z0)u4jW|f`9$P)^ysOiW2hV=4l-JuAZImAOKkBI3@kofiMXcKmyCrde-hP7*XctK80 zYfF_w2nZVRICEW$82DZx_B86V*D)v5oc}eQ=ys5sweS}G1U{|*NFP$N3hLI05O48r zmG>eC%s4iZ6!hM-%M3HbhxP{8v9l~jzZsj0rq)I2v#TMqkm4bD6j2Ef24=?nY`@e@ zXG{ssu6#woXafOuu3As2vPb~=XfyyxGNXNbi-pgNA(M)vKC_VDseqdf0*7NyOxrto zpMHNNJ|KQ;#g|_cwhDVSDknEgjAE*0;ZyZJELx;1)+WvGgkS==Qq}4?n?Y$e00vszjx=ynI|3%G>3+V=8 zFVY8rf&HUw$Tl?mzgA}}-z zr5rGQLkGoUo6Jgum}}jt zhlXIc^{Z=2EI;Jpwt*Mku)l2fdRH0+ZyTe>7>7}DdOSL?1v`KsyoD&~A&V&Lh-~qm z|J_v@0;Bg;(F<#Cs8qLg18hftuGfjGeSLrpo+#YftjP08jiw2b$$H9q`tD0WRi5BW zN{p#lL!ry5EHJMgE&HnN9%?~@;};~+COuh zpXR>^-4WmgzHup9Cfa|LoS;P}jL|C6wEAW5f-keg-M)PSglgI4ONHq*I;%+agR-QZ69x?8Hmr7kuGWY6&y7$l zZA?6^yWb#o6(^n~UCxQ0n%oaG^*M^+hDinA^moiyNF-d1ZDd%$!_@81oX=wppgnTH zb!B?MP&eP<=av>a+t0SO+K(zDY%EXF;=7AVr@D`gMg!v`zv_WZ9P?shV*m@7eEyFm zMPNEuE!gwEHc6+Qus>Z!|Lq64Q%TSZyUn*p(1lbL*L>lSK18 zcEB|y;~FY(#``J2l!ptb4H@^L1OPNU8eAmc)=tC1ACc;u(t}bmOt({5`;({lQzkdD zBysB*)yv2g+zDdzu_vN9&@Q7O6IOU~)e3Q1t8S#_39Y4J{Mgh*xHinFkI~3w_l2L4se%|<{+qW1Y(!5MUeZZUQ0CFXo^Px5Cx~JT143;u|7ZLr63fRj z@NK`p&!r4_B)j$F+FMhnnqtGS=QJY*ZWGd-5p7kX(QO&%=%cAY3jfkbFM2IepeV@~ z{i%mnEprCIi8kT0!W-_L17>d{rt)QP$j4AR*|{WdGhf2sUx^)1qwD)p=lSp( zJ{#Ri!q3G{Pk{@|)Lo@if`M&5)TKQl1*XBmkB;8kN}t4Xw~X5MYLzzkzv%9~1V4;ui5Gq2r`(_X^YEQe8GI=t2|IygRu z!xh48hmCKt05lBXEb{R!?D40NR8V$Otkh4<%%ChH>Eyp&0g5M02K=ty2mx%n9vf|+ z?OCr0?-ScOEE9o;St$T;{?34Ck^%$vpp6I7@BFk-g8yc#5{Q!bl^8~rDSCHbWy#+C z_lI(pM4=1n&$iHXYUzLRly8ZW#kUl~ISZObqNjNn@JTutru~c1A+>*ZzYTO$ z1H@Q>-LL83&CaFs)@c_BBsBf!AP~U5if$K941HR;)dCTTID9jF9Mii)cX%+lNrmiIYUXqm$yms8QyCkS>StX@ft3F^y657*JF_(7%1Khke~MVsVe9iIPvE^BsVuM}Dcaj`Er51F#ww@lw0k-Q?`7Tt;yzHh(4LEa zbR`$i+TL1qKZXgnK#{Wf{O(+Y;~|e(ps`pZ?)#}KN7PLn0NcjpC5DO)0=MGUi9`w< zhhn85+)5`lUK&EJw;sp&m-t1ZUpioWR*wxILVn-8F#XV!`5 ze%H6daMh`Yx-oi>``9~2uR>}`l+&9aRS;zGtEN6ShgsAeo#L&q$yJQ;c4I+6Pn^hQ z@Yrwl>nk5JKt$@DCo=8C`!|k+e$x-H0#Z!N!(1>h4~RP(02mkGy8bog+zT1B6ANft z)6=@}Ymc{b7A$;3F(Yd0Me(H@@8toYhnN`(YVqTh*J9e|O$bVhZ+Jh?$f8Rz<>kGN zW;#d%1=Zf1uk5oCU(&jZ)+HI|H6uYx0d`8+Qpk>nnW;2D27XMp`uDWsrYLz`qU`dQQFbBc z69$X}%A}NaKU{p&sZ|pmm0$Wt*Nz16e2@O|MEz6eNxdE6UldZj{VX#4M&;2S9up3E znkNIY&_z?*IbFFD#r{K`*J(bun<@0+t+FZ~5-|B-;z*8gQ?O)h! ze8d30uj6zlfYwwZ{4IKr09>4t9ijOECQ=nP5zdaaW_2m0Ppp%du38&Q)ya%iVE*jX zk(tDjz7--BCN+%pj?iaiwDhzumcMVpJBGjsV*L&NksBUVf} zk&@CY!I@9diK{fyM7{JYAM;dy50P|J?_OIfgh>K_~gRCD4u$zhBEQcL)M zQ;+^l@QGe6xrAXs$`G2~l2?eB}KyZHuB@e%1Iy3^dJv;-gx8%mOZ4w0GUn z5pcg9Ig#_G*1ItHLsNWt<_SpdKemecl8nFF(L z&Bs(Xb_r9*e)S(R-E6f@IIhqu9-V1k2J{o)4^E{xze{gM86so4c996{MWfK`!ZNf!IND&hsFOqiB z#1YylU_kK8Y_sM^bHKGR?2mZg(QW)H$Cx)q7-MuTO>HJcLGn=xTb2}2!}oN>&kUIi zAac{}8}mmUW@q>GXoy-e3h{2lw=hWV<8E{a_swboHVT<^kK{uG zL08)h01Ot!fQ+xq&$fKBFkGntV61qN`!62t5>?>EsA$3VJT-9UER=gC&I*i#qQMpz zQDKTD-it08gg=arl6$pLEhSG z+x;__G5sunLl&qYUp38B0eNqL*;VC;8vC1}^Iv}D!j(~pfVmEV2l&pe!uHZ1Q|zwL zQ|@ODeqXP3&Q{qhqU%ZgYJF&BUBopUZIV`gzvGkJ0TT53v!DOsrekAg$mfwJzEw(f zLbFk8o3v(Wr5x+Q28_E!RorxIOWekEpTu@fKx)YJxN9avof~zy6@GG8plZSs(zfk3 z@CR1n#mUr>8S8EM&ukmiD>_$|7u>z>BhTboSJ|n^20cswS z%>bmmaSfjsTO)t~zwU@vG3q;quEHk~rq4;{pBWkW8!WOko=Z5n|MeS}=7cTyLEeq& z6g!T$&j#`nfzl9I64JRCLGPQ$%POldt%!&9z!xus@D58jCLLeUg#usN5@=5h((99#6r4bL)>adg!^!^pgYVI6Sm0 zl43bg?Gjj~LN|$1H#5wyyooc9_J&#X{$q&ljrsKrx7oR|EmBSgi(zo69u&WIR2CL? znXrU#3c=2A+^38C>Td-tsHu~@NP;zVxHSbyTpvW3UuY9t)(s}#**Ow)RPn?olgS~(GO&lBc>s_@8bcgBu+p9uA)Au-Jx~&MAH4eOodJ3`9HmP z4sk%Fse63C@%U{&U-Tzb;Lr>{|z70<`KWy$(*3m$v(|XT-3V7?&%N6=cs~s6SHaOYJk9@SY+l zVF)9%k;rsIR+!RVS^I4HG`&Tz;0@~j2HZ*KP3$kK253%8an+pgf6vqnps)7Gj%JND zZPC|(jD2vjNpeB2JxKKrfB1sh$R6R2ThV+cvh--TfQ_qEhde;L>PT#LvDvMX+cj^sy4+r$AEHEb4jUuaVFUMm1;r}Mj^K0b*Oa5~{!Q3> z$iiVNVTD}Nop8}f8^8{IvClZuzCJYECOzi`h+?m=JdzZ`o`09n^(C1Qcnwr7J1Q~R zX#w9g&Oy$Zz2S#|@h>>Dx=uMw<>VbMg*G9YOsThXa^;(= zB}gHszZny;JYL56^H86vcj0S42YcCmL+H%EkVQFnztFLTZOgffv%F6V7veivp}_`S z$3)l3eo&l7W3u@yjn5abh0nFxe3 zvXq1|o(pc$p8t#o!UsZ9-l$Guw(DH`o{53nCS{j0?Ad~hSYI&18I+Y$@KqBl<^Zm^jG+5WG*%$^Z?{!CjhT>GhtJ3fllAi8?TMA*WFIoUQxH9CpFcQ%f1zD(pF||C)&-vMm8BlzHVo{U zLmns(mGAMvg{m|BBG-y}m44kJh!+_1oFk0qRZ`h%AWprYy2yQ_L(Qi^_&t|LQkuJf zKZgF7e4W7b)zvxBdLf>{^>Jdzg$X+EMw+h}h7v;CB7yq9cR8?eq(D@7cUzL15xto~ z1S8r+p9VKyU6N{4A7x#gW~*^&*`7`&UesRVP@Gzj9x1|v2f%izUg@$1TMjZmR_CDR zqvNfH?xX7ciTw=p`G*O^jV*mw@GsNfOSGdAAkd)b!K2m(*;K> zv5J2aV<0sFk~DeW;pi6ca4S8Yg)>x!J&#=M*#rNHebe5h4J)ipL^VLLCD%3>L_mT||ETM&%T z31PO;&T|9DCg6-5{4x>;LLaSC5NHnFsHrdH)aFq?IZm|8Dc{G_Izd;>tgcv?7eCzd zGt6OMP)C?=UoV*BNfop$Xcwnl7d!}Fu3hR2h2%K=8qkF6xlWZ)_Ua4-BY)$&UYcZ) zPFb8*-OHMYsQl_lr5OAP$FcK0yNho_&-PX%8fhQt6nFx4yllf+x`{=~N+9!)gh=JeMutuG3u0N8pPk}3iVf_-2DWjzAF zyP={#VAF|yMNtGOQ!7RbVXbg1l5eW)b?@m4o(X?7N#Al_du@NC@fTEEZpJ%0?rewP zI&XSzhqD--stmCJI1>CtfPKsPfb&3WDCUM4Sz8Xa5YFa?Q#_vG38pwZ7%j%n4+sdTqnu!g0hRUc)$kc?}bRE9;Cgch-}&uqJ8-Az_XTvXJ_=lzdv2sGmQVdU~l63v_X!bMHGt`VK#L zTQaG5YHdfam6>v#0r^_$@m#dHl8dBSw%=0rCzI)wakjR#r~Ofy>%K*-nv0Ch#~fa(LcOi06#j;B{Is{zj%S!%HG~)D!+J8nAlyRt14zp z;V;DVO8f(fO7rLM{Iz<&PS@0p?2+^qh7P`-%PL~V0S62*IoY_(KsSYjYG zWe|*x{P^zewi~ObZXuUcH!t5~bMv=ZyW41W+qRNiqmzW$N7$`jD#xzVZuq8ZjyQ+c zlpI!}5dX4s?7qyJQKD2;c4NuFK^QT5ftOf0)+Y!4feqSPNX#zN0siQ0nU-_sbjU8) zNE9#v*|i0wZR_fFeu`D1z26 z_(GxZ>!54F?dxNmV91?gmCN&|PFbSG3!%bi>P8G*>j$l)0L9n4u1pBNjMI58jBt6h zW@2$9N6iQj-lAv*hMiuTgB*StEb(hB{00^);dGwatN!lE_Z7~_?mMi?9@*PDbp!hz zlzm%6m83V(QJA#_f2?LA&t2BvI4f0i<`X}>T6~wBPNSAXm6lkzX?-CZ{m}x}=Z=?@ z86YACPA;)nl7FnN7Bm~?gQsp~ZV13%KEOkl6AhE}&|a4l!<^;S(K(|>5rfSfpDaNj zlhtn?bxl*%6P@0~?4azX-9u9MH9EQ!YtJ&V>*fI}7(|OHodrM@RDGX7-_+1>Uz3pQ zBE6O+X4)73)J8)sMO|;c>_#-E8Bu!Us`Y$5(68v6ho3N4%H!DigXbnL&zBh?NPYQ| zsJQvmNWqrq_(d%O?IoZoJoktItraYg-j}fhPTsj+L}Nt{HZDdB5#n5!e)k8>1bfGS zb9n6%vn?FV-cr~v?x@W2U+#UX>2Z5=A@}RGQ(Ly@x)q-iF$|;zFzzAfS)#jSH6Srd zHq<0$QDk8l36b2XHkR-@Vcx_S17zP?={>NYI?)4M9aFlB%=D&75N`hjpw6`&&ps!< zF^QFebT?bSq1t$CK*@q=w9=C%2fLV`+Z{>TrGpQSH<{bFO8e*PIezRW_+IikpdEVP zIg=V`3*`o)$N5VAOHHnE^qG~--ir(aVqnL#^@8Yx*9zcZsG302@oSv%BFjxCg5hXP zb<^dYvXpRSWh3XRJF)xN#|O25w1L$inXfJ^mv(-^+B+d+y7#YQCl8f%&0=M{hOWjJ z-*nF(Hbx|`tN#3e%eq+(g`1a@Zz&u zG(ScAvWUU0)1tnqU$({2%nL~?>a#orD;Fam ztR0k1m1xQT@4h9{uLP}dAOTvJ@BcO-wz`-s3m*s+U~KBJAaA8^o7xW7KzS`O36$;q zbDkAen%C*AI%a?C7QBG9f2k0n2)K>x$E9d~R?rgqyYiWQzW*bIW{6r&MP7JV{>t@N zi(+oLIITG9@OacYZUhG5N?szV@gXCwz4-!9m_Vp%LO5R0icdX(*!$FMArGRAB&7f` zA0>?konj4b*p2vRYtTBTjdz|B2S(mKoDbna2_PI8RP;Cd!A!tl`>)H# z$T=0qZUSn@3Uf=Is-08g8(*~#p81D6(YN?@j$6|CU9%hP4|pyH0^i2LR$~$9fyX5g zCxF*~K%Y6X7;XbiYB9#)AH!_3IhAuHoDxA;Q!1&KPcI~?^N zmh^QCFmds#sZM(wTZ#VNmTdQvF@JBMX}a4jw`fRfH)o*W$Gc7d_CR3MHF4;ex+SBtY576|21oN(g3{(Q$&L(`j>+~1nn7UB zIs0+edjm1(RWGm;h|Np79H0rJx)7bMRw8j<4-cRT|8vH(!4qtNOTF= zs#Tn_FXxG^u7;dWIlSxonLqb;gW6clXv~Y>aQlR*k-GlYBb8neC1GoTHl+9OS5!*M zN@wzhSv=o;PC$;<=jT-!OOkzAf2wA!4wNzj80Fq0CN{}-4n^QyyQkl<4aim zUoNz~D6sJAAb7H3Mc8N+s&w(|r9J>$bd$zK-Nz6dc}HFZ)B z(xN_A6cIh2xt{$MfP9rNHw$j?TBHk4 z)&+g>YQw%>0KY*7?%hxtcUl(BLHoV!!Z~XuaGN4KV3Kpw}!QbZJi_bJzuP7Ew}WD=)H?j zasE>~%a6VImAm)eQH?M)Sm2X-9iC=F*cf$8oq;>;lNo4I58k;fOP0gC5W7uwwl_l3_=p{F_+S_y$o3*3Mh^r z^HfKZqxo$nZ%hnvqs)NAuK%eB^kSFhMiN)8C7sLq8%yo!afPF=pC;&V;W73=oo`gwrahe_)Ms>9#UK@J zlQPw5*M4eEc2VvfpIgMR#)CSA`53n0PA;YiE*!q&FYC!+q~5%Q=qnAzMkr}SpH5~H zd(S}T=H(*?q=eY6t?|h$=bjl5EHhb^ssPqhk0%~g7V_&df%HCsSH+ElpiWnsXwJS& z9@dzDJp(z#8Djv4#k@+PdBhcC&#ynpf7x&uvy>c}SINcyMFsQT2~(;$S{T&4FR>Gj zeBjF9zcJO4tA9ZzxW!jcS814{=}xBFN|r-ICW^Q|@+XixW(AjIbrW0K5^(AEXg9%d z&R7;#h}-?7qse~QFEHI?qCY4%XJAT>z74K%|IF)ZxN4+@?)L|pElVK$+u$QN0bMJ2 zb-Fak&fPj-7(pLLGfG#p*6N_P`LQ99$O(q#NU|q|96iSGMJgZc$Mpu6%|3|3|IOr2 zmF(e)!Q5*20_2kpBQGWHSva7!%?ay*+Bb9(*PI0jQucl5E(Bl2`uW}-RcGB1{sg_45uD9(4K9RWcL^w;8tfsEcP^}ux?1h`u7|J>bFxu z_#b}aoKx$0X%+1F@S5zV*GIeccfw5LrrV6fez@VRR|=NCDe(EYEp z`L5%SIHcg#j+Bx1NZ)17WF>vjRjL!yGvUbjy5mC%rl3-}B&vTST8k3A=AE?4#A7Qi zsqHTENEF|QrB1LHovE>yn@erDa4Bm42oU_ur2<=hzu*RQSHB^;HG^#Y14oxqJPjCD z;|>i4;8h#zN&g(*Gxw7MrPMK^-}8@cErP zs~4O9EuDeWFaF?0lmdB9laaFk3C-gG?HV9&6iW{Mqm~Kma&@>vKu#pT>Aqn#@^t3= zHpbRpZ`5)}ip9nw+cn}W6BoSi#sCJXKSW?J7b3IxYA^k(^7kJLle~FmB-^1)lTq={ z*!YBxQBv;6JbEJpGuRo4g*!u{E5w4_I^+3?S=V+Mtd$6$2i^_?pEWLC=W8Uo6$I&j zopB#4Pv_hq&B#|Qxj$Hg(NMI?Hct-f-LgaSq#SZ+ezdT6Znp40{A?75^tx#La_RDf zcrGYbdZpNLejoYwb<9$2$a9oK>mCzXtdJ{Q$(^?!$HVY)JpUEc_i@037kc5)MV@i5 zwG9|;6n`1%p-B;>eQsUxEi|Q>&)0XO9g^=N|xBT&O zOk7vad97}(zof1>K|RhIcMTGB?=N*^ZY_V~N*t zkX%25+RBL}tc90CXPIn#+jqtGsUEsuOZI3Z@d8;T2oF)mBZlR*aI$yR7UPEmr%`?a`dY7-L?PV^0 zl~Lr=!7_X~B+c?Ed&nle<|8vau*N0qrA^t!fM1*T;z~AX@hHuY5ur!hBlis8-SJ-@ zD#C6&3MhLT9M|@ErcrmQhHU=@InjlAskw zQwWFpL|nb%m<5wza4^K=gaiahqNg659s+0-uXKl$%|fpmE>Feq`=Jf-^+{dL8G4im z-5*6vBSD=a7}UHeEVvt=OuQr>gg+NOhxXd zj_DB^OP5o~KC~+{4*zN-di7c_PKEtSKEYlO32OliJqu&Jqm3f}uf*HeE%CA$JnSwL z8#ykUy)$*_6>RRF*9CvAdP75o34PC|)TDB<2Tno-a72X{!NK=(Ds$xhr|kEekGQ3M zN_A0$cjWgC#!tCCPNtxaa#t&As6!jie%Mf5+iu>0z4H*1QUAtjw2JgaIx29j+hdOdAbK04C*czS%eKnPyBK|Ylkt7v zd4~hc$;vhhVXc(yR<@l@=mdMsOR^&;7pR>|@QEDF#vHWjQ_^OmUy+jGcFmB&HOyxJ_Fzd-Ay&WC`Ls3Qkr0=yBB}k> zc)TVy$Q!brZoCCr-w|`~8+-}Yc}nBI9iWpiF;S4D{yhyYcf#KPfry!AP;wi~X9zV* zv6U|XssnJ7NDdh=wO*|Q7E*vJ;lOJM5pX_dJhjVY5wU+r^#F!4bz@ULd}92if&_w` ztpY-}rn*arQ=NUOv( zWcZtwi$`8bi=Mq#b~*I$mJNi@Un<|~!BB8bE*@5pyO)O+YAP2O*~h7omNGj)i~k4j z^pJNIGyRM*;s;QrAwYpvX7DiB&f;|xY^PQUk&G%Y-aH|{ia8Cefe_OAV@8cBN_*EU zJyvbQlJ}k^r#0Kjoi5+oZl%&*d;Dy9`v#cfW4~oH$qj{vcgzv-!JJ8vgOouS*=ivyUG=-WBpc;=FQ-<-*lc4E=~JGkU{; z{PoN16%jjT`~S57Jl7`-Yq$t>>uXIZhU1O~lshrIv7LEsT=XI0uV%8uGy3=Gt@R?K z*xZBnE5P#wLNY~X+i+Lqt>c_S;c^-xb&4gl+r`YaM56KZ>^y@_2Y*MiSK{0NP@@!e zma;{!8Th>WEWP6b0VyUg_m7A&NSyd=L7b|Q(b@jKd-JWBV!G`w;G0=k<08BqSkK0B zeV+Fs22D!0In~ytJ!te6mz;H?x^+U%Y&z~+Qk{6MoYc}BB8@=~@9_|4^2z{itG4CP zywd)@ausf#@Y8u}N;3;1x`*}Pv)t;EE@aH!oKyf8bLFAX9<=evfwS{TwqLp2?v34| z-j17$Jov2VQWQ%h-l6+ze+(ZS?BK=Ab!$$v{&60U3?LPML_7Pg$Xa>Rfl z-N0wNJBc{6Wxj1J)SP=XcG1q;Nx2lWE*Uk?u9p9qZe4*82v1x)i`Y6U(hXj+MgNl6 z##7^em`cVxOK`F)u_KcV;~9~l*n(xGi6~kIWk(C-1c4ktqs8%GIXOQ~1EvYdfhLsy z1~Uhe$Fo(&M+R_64#@Kr_cig;Fu;&ug&zZ&n~fmOEaMGgM<4b`>AdH7$-x zZ6*{TQ)J0btVk5|Tx%Hka{H2=53kVR5N-e%2#ub7@GSp8C-w^v6twz$=Xo~HbaO9e ztmhDe&(!+$*DaSvZRvg$QVGhQ)OBW)ypqx*hkm|UP;_Zwi<+|}!O=tw(z(MRgCSkm z`x!dMbTCBn7}mu{Qr^rAkmP2^X0F>!bpox-=;tY_D68M%z-~~r{+EZdtQ3fIseUX4 z$XS5s*1X~Q5fRX>xy#ZYkGKU(esNVsEQ+4dae~{}(w83vqR;uWHb%l`g#HqjDVd|SaPd&x=VU$)Kl~GedR`i)saH-g<^!s2_4DcPp4IRl2BGuJ1%=SBqVuQ2czjAqe60aUxV|d>hvL- z_^Ic0j3-~Q80keqNst4_f0A)9t0o;#JGo8v|RKL^dNI9xs06OJoI1a+|1welOawV)i3szq>_dgXxRy+ zS=Ok3n3lH>-I&8c>$Gip`$AKb6>?KpBkTQgs)j*AeEt}yf>1F=CLrf zS=avPvy`$A?u<@i5k?ef8~9iEFLDX?Mz@^L_r5Ju1-+XPsV00u4E_wub`*U{Vip6VcRlCNXulMmYXWr7szQ5edx9v|}Tmda%s; z{l%|V@f?&ERS&);Kn=f-)J}NUCRjIJ@mNIFCiZoBVnQ5@Q`;;XijVSxP=Z0CvH;v# zI>Z!NuSZdPsLqWIfw}iqPhXPHiEhHt2=qdyM5p3T-YLCH?)pn+rlmXd;B~1w&783z z1&R__Gp25;4EW&JEU!&5RGP9Ot|Vzw)kUF&CHY zBof`3lD)lmC?_>MiKuJlT*FP&0Pd?BOE+|3>Nm$Bq#~V2g9*S;q2g#>U@z0MPDPe` zxll&6vo&`?Z-#-ThPDyF9ea2o@EX>|V65UH3q_oeC+$vgoLO#Po69-2Y8@*^ zUAqY8S)QuWrK-I%I597;zvekC%_}2$iqTc{S`-m@gE2TQesDcKR7fu>` zx9JGw4*SR_o8a!gMnX9^}Ka&n0OD&ME` zH@BYP&jNd2?B{qvvxGcp zkG&lSun~-TV9@N3-#P(7ziw|~Uj4>P1C(1%RkFsjh6`$S6G+oDCcN{is5QJ4Nd%_7 zn|)!0HGLkb4PIHYjH^&ypCE@^H6xG_U1@|5ZR^dS4768kA2mDs&+m`+tRVxfCKjz8 z>>Fb;9&Oq^(=a32wS;r2#=>WUQ&3YI5p?7*i%dY#d%RPsff{^Huz2^JMKNH-T$;PAurr7(2#?c zuTyYq4)I>?0dCyPKH%9uJgXxP0`H4hW{zhQ% z-3VF%7M6D@A`+MMXp|30wm6ZU^2No~ha!wgFOCN*5N|~0l2?x}TVak}R+p7mLf8zz zOV=3B55BG39&f{+1)nsp1lEGX+KVT*4_Sa+7lHc_(M|l1>_3FCBwW!%ZtH$v)};}u zW$6C+F0zmjyhP?5oYCzyV%hUPm3xv)fqpOoKgP)?)b zyJ)|%c*gXtCu79sbjd!lxc_>;W6FWB`nu0I}LP&<} zL*r`Fz5;`_Y)Yct(_#yJuSC~4^KiL0rJ}c}?`*_#zf#cZa%bnr_zFQvDk9Oht0#E; zREB*%B{}38^5(g4cjw#tf)P2GuaArTes_ohNzZ>rxb>@$Ely|upYhM!C`jn{EyB0z zvPP`z)FvCNp-2D&p8oXK&TRyPwYv20!x@%Er+MIrT`*ur_E$GS|^7>DL8ZO1WUK<8-CgCW1O*}r(hKM4B zg(X_6=eWfy|KDj7&E6$Pjalapq$3!VdLc z>2tO|XM=0pvz2>oajEOW{4uSobE}oeHXY&g5&>W@R|38uKnrh1?KMGRYeUE8)8kHa z^46x!j?x?NANe#7-zCY?Jbj$uq~S2{gT0!4_NQ^l3R@G?19=!+{ClCkb>|i|4)^6{ z6Q4%#js|V%^ikT!z*S6q(z|z3AWX`Re^3pk?R!ROu~`9fiU~eZ%Wca6LCqUnL{|oh zjL6YT==-I%C`hUh$z#Jckwj>Q0A~&iE9iyWMj@0oO1^KZai1S)7c1l?q^;VbZc)-G zq3cs`5akqXtq$zMg zdp;@Mhj4Mg8+fU8n722Oug3X;LKV4vlg!~#D*bFe{1rBPMfJU`+VP!c(2AsPM0(2U z)F_lM?;Y=H9^57Zf5K1oRY&0R%XKn6w?PE<jx9vz#*pj9G9)dn|VEbbn5D6OLj; z$0JZKn=gi7MxZs8cG?Zvl((75A7m7|s2W^J-K*L;*XattUb3G4usJUh4!CDVc#`!v zTMpzbQZO%rKWvmrcFGN7a6D6#s-3OoqR)Ad>F;+-BtT1u#LUzX62hg5y*9bC1W}YE zzv(V8->@LZ5L_Ze^$3C0%_Hxup_klHIiDdegcl$?Lsl~el8XY99n0w+@`t>KMm^d0 zw)x_3$|}U5UZ(#dw$FV1tE>1?RTaD6^ps&f@+q8z_Rm#c19FtiJ*n$DUjHKJ>@yUA z_0;rr0}laK4)A2k!$33eDTbAi!2ON2SZC~R?l&pHI}rTTV|4SpccVFWa8c2{=N zh{c~@upzIRUo*b1SWEal*yF^?BbY=DjdGqK0nZUn9+X>h+FD|oJ7q~}?t9Wf{4d}0 z|Gsr9HT9w(rx0Hu(eCze!R(Hqs|K1zFz|0N>!ac%t!~P5!RXg#9upTVypHdKNk~Ex z&g0!qc!sF^2uoK3Ydfo6*tqKK8mn3n4xpGxi*8LK^GPC}uNb#1Kc6U{`ZkQ#Zl#I~fx9GI;_HZA%HPz@gue(M}OKA%^;oiT!_U;W)qLt&zax9?c* z&dc_y4$1R-S(YtE+Mb_%6@wf-@N3#o^*^zJMy=FovaQpJI3Ob7UVFInn~~1BB;Zpu z#njUrLsq!nK=t#L8hL9re&-bo%AfZhdBy8?&PlHRAN%;eg#ve#=h_P$gW~lAjJwm_ zx1w2zE~V#TlN!#CG0CedQPGD#Uyt5!avg*iWb4^ZO2@h5%ehGHG==TytM9ZZ4v7-p zlSCkob4_8;s=dv<+)a4~r62l1qfLK^V7EYy>31Pa^*jIH)ZgCi>%gIcYcF!aX!piT z|Ld<0w{00w+mI&My7qL;*wU?n>}{TBC-wAUBtPZwr9=Jc=tAm1R%1v?AB<#>%y^z~ z%%Q;?q8fUvSFC=_;SR1KlZ(sz4J+T-q{4KMcSH+kbCA%u@R{!V8UYa$uyLBs^&L_1 zBNZ_R^(TtM?Bz#>=|?s(ns7p(rwhZr*e8XumK3(F%qQ!PHW;uS1g%yZn|m3bVV`86 z{6BqiYrR7aYO7I`HcJ2OsJ9Y!mD1AsDH>11EqwYtl2muNhIpEbaXX)1;Bne8G+DF6 z^9!ZFdGzvabS>_GmMXWBuzVM8LJ=&scWJ6nI6dV-t9s7cODm0ed*2k($#Yf>wwog@ zZ_&h)1&B(|xEiv6x*~<6-w0@jvt0XC?;dUlWxdc|i~IB>@CD+p!lOnprGU#0;ZO}H zS}DMm2}egieeY9w`w`@_@d&5G?WRNn6%5T>VTVwLZ%Uf3&BKos&mcL}WBbn>58iOa z;2UJl*kmrQQypbyl3Nq$g7hKHi_&^)3$k9$rZ5&-^dh8#6y-Eg#P z+_w%25O>yd2`m4|e$^j>msgi1tB$wO5QEIiL)eGkb)ggk9K_&2JX!Ls!%~nHWUxeh zR#9Tk+~M|dxqI`zmtsi#%TG$#ji$d8&zb#4WoquD*(_2gtx7XiM$3iubIj&%Jbgd_ zbj9VqN_Z9aKoHQ<3>LRlIdY3HF}MD(}pbjqnXdC$$#{qyf zk{8~Dda(Z{Dw4~7~0=X7T23l+AEl0+arg$Jo88oeSSP{~g zxQYQW@6l@7rwZ9@!M&?*q9zLc}3|r)Zh0LvtG%!N_J)AbjJ z4Ep7CUT=A24T8mKauqi*oGTw|npw(RYWW zsY{IS1L7@>BmR1Pfz+Ew2q+U5AQ2y^r+}9^QCM&l3bHP}Sj^pcSgc*VjnyxkNEQqg z388%m_?{NK$ERO zwM|}^j=vjA(=U1Rcb=JncI3Z-JezFT=I7;$=^<2!VF_~SsOPoE4-m}!hG9fba* zlA7Oa>S-9^h{4AF3`0N6$2NO;U&?!Qi{GbVVDg*P6HGJ=%*5f3t6qz;_Ehw!hsRDtT=O<&&?vDoYeg-1x5!R3G(G|x>ZDsi>*#4y>F0DC>Jh4l^B zxuj|yk&Q>A(Vo=!nuZ6vDRcP4cWL9{|rZzg$r{%!$~IRJzTr$xTH=*o55bj z*q!^@XFnDGd~_aVezqmEUMqLikS?Y0Bb0;dKAN_tJo(pl_4>$|9a5I2(Ff-3c-wcq z^ixRFg}KUunZNR+)1}eLhBHOhMD8(~b#aZ20S{{#AC1mDjUIeS_3plN{hgD+>~5bi zBw1_t^T?#|(dJ_ZB`KU(mbx%?-27rWz25ix4127aJrQ;qZ;=6hru8es0@Anm>))|- zBXRj!kIvO0Hx4-r&Z5+iS;HXpwETsE6A!IJke7f{C*?3Sqa2#SE@Y8~SZo)s!&E{5mcvDe)!?caPU#PU_% zYmQ#&z1=VEc=b&8{$$y2Dh?z+Q_*|J8K*&|*3u^LHleD01ON9H%Vgby2>NY}d!7ix zG`6O;Ga+P%9$(l-YXCPqpG*v6G#OOTheFJ zYHsqBL!Uy9rN#>KhcatgQl##O=5$mP-;(=7?%?9*yeT>mHm00b9e`zYb=GlKf*;BY z`Ye)AMf^3`m40RYg*~jBxie8QORK8n<7eRRnU@9jl)VvB^$Q=?9!qfuk4S6Hy6JgO?fJ zrm8*+r%7d*u!$_R2;9wmUFx_?oRr=W8@x$#-d8eGBBDd7sr=-V@l~V?L$zl${{4F6 zDeOc2Y;Ed|DKDJB90JA)bWasDKWeG@^e+03eY>$5#^$ZWHNMfC{HuB}F(s3M`mf#4dzyQ}y3>xQ zzi;!V#{a2~x34jKeBk0`cY2tRO~`hq)IKjLbF=2N#BEaf$7-gaZ|eFUgZI(qVe(AC zpZd7tU*C%P{Janr(#$zvFsc1-P!|s&pcZ06U5%X%THM+1vXiEb>8zNfN07oLx2f;j z4mkEquG}ULY4>mEla5UfRc6Ug98<9}|8a3oP4T6C=upY9=P?44R1sC3--fXBm3L+M zuyO#69-NdQITC-ocl#^HdNy%JQrY@~lVENcHok$#ZU-hdFjVsVh&LFKX*N>pXB0?N z|8OZu`<&N&slg?l4(%Zr68Wsxell~P|DK4xI~$_!JCh}Wg8V(*8_=C8r{50%eOJ-gpf2qke9tXf*kWQyE8Oa545H0g&yHa!nb6`pJ`k zs!ZyfTaCw_v*Il(1x?Ro$*nFQo-Q$1rP>STt2gxr{1(WGYaP|Y@cffQoHf6kS}`Xz zA^nqz;vfg%cw`)RYw;CAljv4>LT{T6lAmt;!cjfK-r=c`v+A2!*8-2w=RO!k5`b5&S=WIvild1eSzyHm#dyAzQWSZ|29o2YU zJVatfsxo7@WB%jr*2NFmKrC@W{cqfcVW)wE?%jdp&XBti5m%QEWa1JXw_q4{eE8Ud zrPHze2mSu}{TGzWIE{eZRCFrp7*f~*IZ_jYt|7gbh`EKx_k70R;R!J%DfjYE?NNUX zUe<@y23c{>fBwgbmME5HuDfy*p)Yr$)3zuw`iSqMkiHXKK+3-*|5O~Ji#js~5DxuIccT@V? z)UTvhVFNtn62t}CTOl9H_D^bT{XX9=^6bX_tUx`y!T(~Pwq2n2x4*yj41;uoy{5ji z6Qv|V^S$#2{_9>ht=8+5!`EZA@5vV8cU;sH8U(5V|HsT1y##YL|7n+-Vlt_?m$v0# zy3i~H`dgw;%wI(T({E27mbRS5^z?9rQA-dKa~59?Ies>U&13e%3xoLve}WUut`F7% zl5PTXo(x5?Y+rY|E)F_8r7S@AXT|%zlgIVaH`&uB_&wj$4O;Wjwh{6FGu-eb06jTy z`1E@&@iloVEv+M#ep|7|?JNE#W0++791>)_-fK7W$ zMN#&`jIH0wO{!ha-nfzgjhF^4jp+rGRN4{^XWX_z@w*-EmrZr2$PzuYjtSx$l(*uy`QI%m^zi!~ zb;}nKgbfmh2J_HdICbP$^_w^K-VZcXDY1;Rp}oh;XHB7Sd0yJ6^!NBSkBhTqmzFSO zNM9jfR17Wc7vWtxWjh(36s=;{cr`4BOUI1&Iqcy7y=k>Eb_)KxhPza~+(QI6siTx7X1#$(E9iHcjekXz>RG@ac@mek2XJ87Bq(rhWA<0g4Cn)tNO1+x2;%ac$n1 zYpQJg2imIatE)Ij=pEo?bJYA$drWq^_Kx^I#b28>*8mgf>@XqiGliR|JN?C?*#h^Y zDffya?^j-L&@zZRD@hc)UHz!u)u@#sJ8|tzA0AKj?qtKVi>B*7ttou6ZLl>~cf~)J z-nQ`1>%#8(sDZ|syGzqypdnDR`S|mHr^G20=n=SEgUB&_nN87nS&Qrau$NlaM5oG{ zanK2ko&5Lp?0#zvh8H}459W#vJv&r*Cc^r^hw&Au;7F>^kp4X zpiJBrZ!Z*~2`xYeAou91lPH~FsAt6nr8f~nMp#aKQC+;kq@vE3+05N@BaN1repIsIYaaQ*4DW_ zF2da2IbsCf&k!u7adw*=uo9rB+!Rbx2ezVWxBiV*6|m-Aa0`Ihgdk2@)Jhxu-=sI@(Pb-o@|k$Jj54J|@uM_@_<$ zbk-EwBE%r^_lx=k9c(%VgK9yb~5cdRluaS_wsKhfqWHuM^mBRv+9h0(YFRDh7dF_+TiF<7gBQ|EL8dh_r`@0MIv5cg12nS&G_c-iC?d7 z{h}MOdRB8Pba1!+O+aPCP)Z|wqkopVZ=+hZ9&(=cZwq)IhgTM%ODGFp_p`2U(U1dN zV79dG%Y!Ah@Nj6h+pz^-}N&)f-oiH1#n{f%|%5316>zBMgm=cI}8|D=J?M)4McG zyIoMUys}|}pt)2i=d8;F2FJPicKo-ppUL7>ZvaJI)-@`6Yqrmlc;dyN8nD3R ztkVnqR?8a8NcTG&Jp$yWlKkvf`S^SWYCP_B0)7OYEg}C9fuEtmR^|+8T~Zx9{KWdk z8CN698O5oH+8l~Z_=%hHhIg`ZaGV6L>5o^ebHMHDgDbOMJ3gk$c)R!1JaT()*$RMM zKVIabi&yYcAg?NVzN$Brn74|8B;aN*V5(S22vi8>@YN+6_@5{~IKK`G`YKKZ0(8<8 zuY0itY;w$Fp$tEg*eE83zY$d@HyLOP<&f)z1>;q;ZlBj$L)_pfRs?-IvirX1y7Ze% z)Aksb`KaZ%3tk>P{Wgj~ZUbyZLO>$X7C$USd>lWl$-+%p^UoP|!&X8sLj0f$z80ChdcB%5fWd zI*@@Wu3diLkZlMY%KiVVW;P851wj!BtLY76%lXF1e@oQmBI=q?-1QAiRQ{(i)+va;ZBc9uu-P=b~_szP{ZEZM(|z)nA+Z zb-1-rtqiZ;F5YG^bMJ&D3kTsVWw?r^m%OzZl0t#sMP}5;`-hUQq(r8cL3zp_X5$ZU zynuBb^KXBpuD!Uf*FVF_Xxp`d(6>bzwojAAj}Jo63Z3WoeDe6vgu|}Tp&m(b=2f^B zAN$`g(ZYGWI*GRwiC%diXd3Op<9Hy+d9wpA9k#J=Nb#w^vJMtT7`o=VY$Vc`l!N6g z8ud4+lCM!ORS-#9385CC{GPr3*fRpqAWQep3FmMQkH;CWFa_p++i!)2wM?38m4lku ziFQ#4NdKC#0VZ_en&(k&vWmB>>@)LEx+Ti2B-{{JbqQ#qu>n9HR&4j%lLi@X(G*c! z0mNnCH9b9PO54!*X4+=Zv&$Fy1xY=e6Qi0R(FWc-HefwbK~{v1BHW6F;LrQ#+4|}b zDNu{yZ0{cKsu$^jm^jQl9@Mf?3r?0eYas~?pF7wk{S0E4ted#YRRuPF0{gDsKL z`a1z0#*G~%D^KR<=|h5gZwMidJ_2`R_rT(DYPMGp1pM+F%6k9Ty(_8j9}`1RzH<(v zf&cyj)EEIk0yAIjN--po&3EW&<7 z94m7-m?>f@jGS>Il>9mBC55l+{=R<2FoG%Hxcy%i&5W7C;-5F%s9sN*sW2AVK-bU3 zLe=fk3wFXaJz~Hz&LL>y1*(Ed)caFj-iw zqTY{OHS{-Hml!}kSDB!VyVSFA^le5=nB~E#TH?vAPiIUoIDv{(b~{jk%Ty$$n;{=H zi5ofzmlO$tpfTOF`71QL9E%{M766XID#<$tA@fvKy~;Rl znzuP8eBF(}5t{Z{nNQWh@4)VG8F=azF?IJ4S5lQkBqr><( z+fTP5%(u%=Krr%9(_CiE`vVl;xK!~n`uw7z>y=hx=M>h>v~%nGmDn;^+LAv23yUQj zQ+nGe7bkGa4L!jf8y;#-A$lI4pmHKvJ=LdXwMQiw(@n?l47`25Fy)VfHl}jOFyX3% zdKId=+mhXKJ`8cBP#huBtFx8gtryjA9IlDqcM|{uyM*eDyQ(=VpRK?d! z(NR8WyhdNSwJtLbjS|I+W)TZ4g?{kz`z8+HKk&Q!Ll>$}S}}Q6LxbSsoaBX6lOWpW z+MlkKQ_K<-tCTA1_VBpOtBogG0SJ#Gm5V{ceexTV+)Ljz9J`*h;htbZZ?(Ae2hUm= zl^04d*kdb8&;{r|Pa{`JZI<>3hjJBbndyeCR)&g%B7Zepr-ecL0f39fgS04&X5( z05%lV)T;fs3vNe$lOly!%vKy;!TGgBMpxzJ?xWZYT7A~Y6?#@XYo-XMUscY!PI#NF zJ6n7a)-IbqX4u1;=~GUkbD3V>tSVV+DOrB~id{|d1OTOrvt(=#X`~iQLAqOG=L)t6 z9%?QTme2dic=H_dQ}5x*-~dE`Q7YOrHt5U8#@h54C1lZ;{b8&6nbnEpbF4oj_>frZ z@~Xd{2;`%$xl@`Dq#LCqs74YA)LXA7E*4Y*0X<2`#~?uZadjgGHG;_(1z67E2y0pR zLge@M1E8so<}+ZX0M3vufT#He7kCs0o7ttxW2EyO8nsd?;exjyF?F3hk-cWwsX4*S4GeqOde>$$PA**t|~ z4<3hZGS3Jy!!frdi%A$_^%|$obE);TA>^LhJ#xY)M{ugomp@tIdeG}8_24<;AiqUR z&E{WQ^@rXQNvXakEU$1pXhLK4*k}@^Kh#e~eCdvjXn6h?^rj__e7qYCP`_#!sY%$D ziXU8jBj8lNLp5kZ-aQkR;;GQKR#cpx;OrC{ZZ>2juX*~PPGcMMf$(Hg8 zM>WO?gZUGNfnrH&oSujZyQMr$5HKtPOIy#O*mg>&YY7<>e_#F^Fp*)z-J(>186i;MmfCLA|9P*?=N3~R*hYrd92+gF@xjGX< z$)>jz#n#Mok@O%yw|DL7{F+bxsuRT$AApgC&ilfBgfWc-LYQezq-boyXi8HToQqEc zwJA{>^nOnZewbzSe3%r((IGkXLUevu0Ki~IkKs8uSo)1xxedNEX>kf3^t?fU zCIKMM13N_kM3xmNx24;%{$A$nw@(DzVr6{!#D4HWlNMfiyN*;BJ=Qw#GSX9M>O|fI znOJ(56IR%rXTZep3V2|+^6(&bM8N7^o9>g7k4*~zt#!H5f`Kf6-N#45R3A)N-Ky6S zA`^=FD0t^!CsNkf1;ophkkj5$JJyV(X^#=B2cnZV1w144*ye;6_cj$rD&jOnL6qIlQb3yX-T4<<@&=r~~rWy(I-!)GW?;wWz(mR%~=acK& zKNIG@MQ6mOa9h2{9)^QxWZf`ezj5uE#YMU3xBh%f5LvzOSckWvSO^K@%WCR?vArH*M$@SY#_{_0=tbJb8#nJZl{8M zQLj*KMWSEgI9XP`1hs(8wV8P`AL0IxBH%4>mYEZ_7>J6z|K-!4Z7h5>TALmY<`7g~ zT1jA1hX~3$WedQlA(((Is1+ekE$4=BF=avGOGQai?yx-GRTDi|eik@pA`%^D`sb$( zWyd8ISFBq&zcD>h+ylNJbE2d!5}5TWB95=K4lbQ=IgF&zDPB`;TO$P4e!;B+I^6*= zb#69%Zu0k+ykp6fe(&wo>hPSEV9bnLXxogHRKPlUAK-^d2$lS3W&rHl&1ES1Y=)b~ z=7dV-fEZ5R)_SLI-z*q4_ZtDQ>WF|NYm{)#R8A070lRoQl*VZP!8kT{iz+Z6NYk%q zRDdsHD2wl_UYCc@Y8eHf4t?!T>hbb8SVy1RR&@M4oHjfvk4A!TmJwubePEv2)Dyz; z$B(Ke(e{K~ga&hn@>VvMmt%oRJ#RdRrkcCK^9eP09^)Y!laO6$WVyG7! z7!r9*%kjBRy7&u`GcsTWTlREs{gIHxd?f1@rBbw?6L(25qT;a;AW+V@T5$UA5flI7}w7z9VfMT({d|fhEtH-ld1Cs*#M#-6R;y?#n?#A8-ut@{?$JKGQMnfb(erT*A zsY@;?S!!9=UQRt{7?`KHcE@hBy8WBAP1F3JIUCm#?T?YHRK|O(1od_FauQk5!(IaV ziF`&6_6EqWytIF%=v)QqiE-PZtPMC}mS&}J5?6{W|1F|nH3qvV)+`z4K2H-VS^p|m z#%8c2vqGKsKn-D(rRoyABG#Q38u_%?bg~}u0|WqQ`m?Vfp=>)gxCqcj*gN)w)P@uk zek4nBL67LiMGydESW!aYt{uAl6C7U;_`RIf)3h0KO)*&ASFIDa^hk%6)kE?TD$a*q z$RkiKdPA_gRh+w0cpbmqv0ac$Jj2f9a1aqkOOV6ktRv#flwb5ao@R&jl!U=U?K(LE z4Z`ZAF$wo)be=ze%GK{vYaz-8zTZ;CZ5UuV81XK%Ji9&&$oUNuc5Zxpz#t8>xMCYP z%h2eN4Z3>h*74D0M2#gY+uox`9$mWT-(d9Z*ByEEY-MU{IE#Fksa9{U^h?7IA-C9f z)_2MVnN5XzrOv+`j8E_H!?<`wY5tC;gqj)2;)G1IYaa(-(IVO6pJ7~u;D*lR+4U;A zN3oOe=O|4mnh)V+1V3kr|7se&V5mnJzIwZ2rI!&qHBcJ_0tE~FOnk*tzZ>=GE&jWkLogB#LFRA!J-Z*fQI5ia3oQY=nEFUjSEDiNp%eCycq5+Yq=kF zR<@mW_*`qQjm$Ux>u75^6}ipsm|yfgY0@Bo%2-gSdb~bj7*kr0HeKj~(9a+Tg)$T8 zeUnGMqT;7#x;;sDqk@SOesV@7g%O=gfD`j-r44uFW{T0DB()6+!f5Z$4iErGPNxsXCwV1RQmer#lX@f;obv#)Ki?tN5pv`c2$%2P;NYxPI) zb3wE9Xae!0okSyLj!nOT4&$Hmse?!?3?`W5|A>tK}eYmvf1=+MHT+|Pu4YDEwtlIpy)W+i0 z&hks&tDI1>3MYk_%REn06Sw4w`GbArT##{YXo%GjB-HbbO%3G8o<`)8fkv$PM&lTaf8-Y<9f=9mz5{29 zb}Yk7gAOf6$$Z2;j@#iODG*urMcUaOr>9#XOQh+azm)4gypN9-q8lGSdk4qx1wI?^ z9pc_ZoXHo)u&*sLkk$I8Zl}G`_?SozNs_>!n~79?Sg&OT$viP-s={UW8TH;LlO&>Y zGC(H7_2Y#`;*(IxJ5NIo(OVgcHw~qS*#Kajt{AtFK(oG|;m%PEDTR13zj3Zao&Mfn ziSxq9L>=Dss%(7l`-u>}`(}D*Ki+iQ#QiznEDF?z)=L3`4tirpvdncewPr?>LwYu* z1M7_~-}^xAnCH0~@duoBe9h?lB&%EQS;Gg$jDv3M*WpE?VEPPt%6d4ypcFrA|J9g@ z_W@umJ4;tt8AI%*=~QUnik!Wql!UB$$xN6+On=x=J5|v()s?ujo@WM>jZSGXwvsjb z-+kXJK3gOzp*Yy!AkRxb`fZ>?H<_OQ4t`2I`}JKV3R1*~z}N zWW!hwF=jPLt+z(tEhbC3h1xVN+O{0%|28To&yC`$&+d`44jJ|?CxsBzf6?wj^nNTRCiP{ASu&+YDgZTP{al<D}q6$U1++>iZ(_`p1J zrZpg_gDBQt-tuY#&r!Q0B!8%0TD0sohGt#X>bc<1CtADOEH>%Fd4`O4@Fg3?(98`y#H?53sW z&pFw?D@|PF_Q?+RRTz7)YPmtGiwd6^TOExtMnL`fd!s~YGjNIyW&Yy%N{+5@gJMGA z#o~7OVDBYfy49>%;D>+&cv~gC9wKNhgd0Lb{9DD228osF@5{j62!hvVQE8+?@uFvK z-Rm!Xr}k3e*6HuJ{zc60m;8CWV}2*`*CXmB-iiNA78I zq7}EI;`XJfQlema>cS$q+D%dQwwC3YZp0PLsSgjO88N^M`AILlUxwS2pBH^^M&KVA5Uo)e`b>I8GBfep$bGLfnA-GnAaQv zlIP)JdADG_D)b;l_F)4AHK@l}ogaU}H7%dhEYBLl2S%foKf|*!v{OKD*C9~G`*#dg z466?t5F(?%jy`Iv6o)Wgz@rZ-<h@XU3}~5ciMyoG%gxzSD+-i-@ZczvbDO00vfyC zYL$H)|Lpj?>O1?%-fMGeNgB7CQV(uGrbkQYR_SHje^W00z`0P3Flaj|qPVyM<$9~P zR6zszWh~8s+%Kv@9i+>OM0iJ%Y{!9@FlGQ=)#uG*g7Z%M8iNQkx+N;P)QPaN z;@rzXAG|33Uq(QNf=6sV?_;$U%JP?=p7R;Tiymha^s8d@6xr-w7;0DE<0b}kOKyYc z`xfzXc%JB{2ve?%Xh$Z$JK^`$lPf+b7Rps-Ri4<_GSC3T3&@{h;S~L3=>2O+!X6AQw4;g3simUv z_}vpiK-IPGXW(&pWC>V`3*xlH<}41+lKLkd?xlf%qjd$ya(q2(bHvw@6I1LC zSeoKoiK&^ithZe;^rtx zngT5rJ$P+a7fycCMwQ80A!R@RcJep*8g!{c8(@Jl5$A_-#x!+o`!vR<=yi{#P#=i7 zL>a$9V&H-$o8eeypYaHG*>#1-*vQQpHyghKQm7MeKKhxUV*xQ_#P+55IJ6o|<6c#j zRPQKrL{K-yv-M_3a0 zNJ5#qdHHAOPHRF5{WK2-y?;ie_nrUXU*cw5r|_@fPo69qU3?DN$O~TG6Ub%A_RlBE zz#2J#5djdq7gdU^$gVbkOTjbSlC%~g7}%T%&~7jpZPf<5*|VpK+vU(?t4Tw zEFf!^Co6P(Cq@Li3HaSyMgsXn;EU@>2{coUf31FhwTLS4#hEDM zFoUkX!$9KSqT~TnrmI9YjidGPE;_1|bz9BNb!76jW)+{C(^OL*Wz@HIL5VCT+A&&( z80$W5TLnC$ltB8b!HLmHH7ST7K)0Ps|801&lBD=cBqSH!Z)K8#i~d>Ee-+hd9@XiJ z5?!$rE>jw>pO(7pTiu zfZinXCCl7KPDA8%kU0iSIF)#$H{%y@3L`zD-`Jip+e+f+591xGMi=$Em`{k32s;Cf z3uX+$xWSX^#P8&;8P>O37{~7>yNE#PO#~bMe}9q%!2?p;+4Vqvxc_#oB4(;lgtF*> z{^X-d#^{$&s0|)(g3xm}JV!{+ggx5$bB6h3S)vjH|9baWGL0!80wQs{qK@cFdbu(( zMtx%z!>aQ9E}%S!;1IRX+u6(+==_wR`+-*c$o~M{9gFm*T5c#nwfasDFh+aU0DV zYVMRg@!xwJ5%A=3cH-x*?9pTr`G(fn?G3X-aRa&GpuTuw$HEZonGB{^Ns?;dVVtsf zav3VHgph^qD{6Wq`pQUd3;+>~9Ok3(Zi1G7Y44VaOg0Y$faIVLY6 zKR(8fo&7D6U@rlxgYvOprxY*r7^AVh^RKdIUp?dJ$WWr6*HBSb8zQOKWAoJPkAu=r zZnwDbAqPi)*#lLh5elp+wb((|!fOIYRwwUQsr_vUvaL!04`oD*u^1qx%l<(SCIZ{6 zJnvvw*lLtoDm8o)`^lq{)XAYIxdfGJTi-4g>782jHLQI=C}6nF2!t9YD@$*oUjcFTZ|GDWAEMSOWfKO} z&&J+3pm=#8Fn>eTM+VwHqtgMD0Ec~{2B5LVnyGEw6j_{9_$x2yYhCtbP0NtFTs!^{ z3UexwV3=2#c@UFN+zeK0Z1j(s83NU-2kTu|WMC3Yo4Yoo*HlVak+W!A;zC~I-357g zzR_z<$_Nub1LS6rH}-z|uhpIc0koxqw2Hug&#x<>TLls`VbDxWfYe{52@wCNr8j{86j}NUp#qu`pfEZL|9;BLu3p%~!d@9mr4Uab?@kLi8 zxkQN{?kvkqk~HaZLvFy6*&7vk-$f4@FHhAR$sEnZg8f7Rez)&PqKfl;(Fw}Huy~%e z#Da&hCQ$1$8*zUqLKSGfW*`yztSPISD{Th|gTLZbrhr}f?uyL0cjmW2(1u0O5~t4_ zx=m}-z012f5Fi8sH-9}2_g4)8S(r9?yju8vN zgBwush5lvC9)6_8TSx7%*Im!kk-p*DLzfC_dv_EF2@IdS=hHHPY#Gc-hEVzo(Dr~=Oyzfy) z0@CmOi%h@g{e>gvlY2jb((SXjNE&P!S_Bw_CbylmEofy?>L=~`PZ_Yr#S6;R;vD}3 z{r(jpK{{H24;BDJiY4AdDa!lp?b6PwG*C7`zyxTXEJ$`wk8aNw5Kmq>ByWHK7-~b1 zLyST+YL9Dz`m4$jD{eduv>?@uCRvBSi{3Rl0Tq3RX!}|Sd$~b@u(w@QUz(T#FIz)3 zBkA8^0<{H02fFVBU(!(k(j0C_0BVKbb}X4@Qunq=+eP1;+K@fbie+D)AQ7&XId*4O?&k1!jXb8G*6)lk0-~)morWyQoyqp*V#&+-8R>%TO zCY}#2#Qky65EyX2lgHfc9Af`eQ|MoxKry0ci) z0{)y+1pcjU-&VH#LI3oPC(nJ_3Oc6tptEd0hh$o$wFH8Se#Zk+^uKCz7STwxvZX3Y z_R$w}J*D7T_r}m(ywjkcU8*~>!H4l>CnQP@Z1=3nL+@T4J@H86ny=#sq8VVT~4bT+tza~hIF%i4_5O5lJU z(07%+p`f)h<#?WpNL!$r;K3pT=RTgEonK%N1Kn%b}kdBXVJjmLK+ zDOV!QKYHb?yg@2oWaU!2I8h#W6B26p3Q2*xC;u_HW?0q+N6c>s8Ndr#0$>I=d)rrS zn3_^fHl%9JYb`%)_=1aGJ=(sp=l2eGKIgmSPsFH1$4j7DZ6Kq`gKO=ID!)*VE~#xp z3jrJnr7CF54%+XUZyvM7)Y<5De%OMFocCPH?bXt$pfKhLPk2F3#a+*SqY3*(2r`uy zAO{+}2h!;Azw1mX@1Qe!VS(LT28`zD0LLS%&Is5_v2*GdR;_F|88^Z1$g2pt`kND{ zr0t86q}aoH@phNI=69vYYj0$@yVJN)#H`6%zd+t&1p_cjnK$<>N#n}6^e!C#G*qLk z-%Vq!>btSA*cdOrEi?8ZQ4SoOro+P#KQiaJS&vP{mHAN?#JdXs<=};m2Qr*Sx&91LU;FbEmIuJ)tSi@;Rj4?v@F_MmPS1X5Bl{IN#dr=@$29fGl--z3Y;%q-9{u&b!c*;Lzhu$-rhS2~rCZ+vGMe8tfmT!~6i68CghRAW!x##7_5 zqmaiAiddt2@C?)f0aU*=4Sq}`0Ioi&B)~d~&v*egsN@YQ;vkVa7{|6hk&-;gO8*0H zV;8xQ3wQ;s<#lWjb0nWKP5HRKu3&elMr=3hT(S$7gfIMB=jtPv-Nd&Cf-d@Bzk*jy z|9`Mg;4T)BJje73>wD4jw+9m5ri4N|Z;XrhFqZR`GhcYxr8p0q*yTj3>$+0&m}({A z={P)7A5E4H_*r`eyI+>>j|Qx~<|(5rdWqBs8C0kFxh*ar4=BvSW#cojR@>skDrt~4 zCUCXh@3A>y67Z=d8;w1=AqauvU~$6*POjSdow7sTVdkBLF_@`VC{@oN0>R{T=yU0z zd*4FG<*ex^o#GHUq=_pH!Hud_NIj@BT&Q&bp$Awgs`z~A`DWCB%NE{tG`L0fkYx=fN{OL#L_L7EudU$mwqeTY{Tg7C3Virx< zJKvk=Y}4{7d<}2mhh5vs#s9l<*+PRmB~%1)sKmP*!;_imwRL zC@tWb9DZwL%8dkL-U&rB|DN7VU~ohR^08W2^AEft z4O+uLKYREpJScMIWB>d{?eU;-ylJsO&7ek+EQKxh_Bjam04Dq9O9AIjYU2@GSW_~} z*jBkppNF&tu=3u<5Kf^0BD;Z>6aFmV0FNu@&>xslMEj zQW~M*Tv5f-q~~;D>tsrUQ51iy@S)U@#X;t075^aWc5L(Sl}(X>RdJdG1Nb2EsJS+0 zgHA24Z>ucH6hTN3z(J=knIe|tMflI*#rb=34BrJ~8IWIpVo`QR{T{xc#hKQlFp>%P zmz==$pP2QB*G>uenMe#9cPYJ+Mnc%SSfgo*wL@5)9cZxUnCeqIs@L-9#T&Px|CC5X zeB>n);+Im3|zZft{Df3go}C z@rt5L={HC3r=27gKfrg9-_SN}YyBfO!_?t>Gh89!OD&zE+Xk+tp3DZ#I40%iPv2)n zl^rQZ5f7hrQ-bRBXz;%YZrpS1u8NE|;c{61b~#NKnqH&^<#wvQd_s7ml{Mq0N%|X9}k@?bmHcniS6PZjJ>4W<||%cq=*5ByTFl3!&q8=2$kDv=R&jx9cCWZfhOT%#m4jUjc}l^DHSoG-88ZSy|E zka_<1)*kPG#RZ+~nA_Ozr#fX;(@fDVMrF~sWVK`!j@|QRpO z)iARwhR7|3KI;8$)FD*7KFM8Q7ELHbYbq3phwIx1IP7!58!^47fd_x}_AL~5a*oXk zfXNl&t>;phe67P1uB~z4&52iKfC-&0zs`GjO#9wqMF6wp2QG!cx{Fx)@jRb`!H-^i z8GYrAebnmc2sOtN)bu38(VWZ^vF4W>U00d{w{1t37~EHEBu8un+k0&7;tL#s*h&>@ zW9rIyF>)|}{}+@El6zM=_KAM9C=ban^t z{I>W8Kq=j?^`e{#ZIpu(+G%+>EFE8mB&)P!H>j2;vlOaPTkE8&eZ;)Eg`096*;I({7OD+4ZZpx zxhqaH9So*NHJ?`hQ2qjw5a_T-qmQFNPy=3JoiWCoYKWdAL&$s5qZ}NI_X;R)4J#lB z;Q!`Ed_VKMpvXFxCTs$qh-cIjPM}>eS6|`hhNlZufYn=UshcBsBsQFiH#pRd%A+Rc zG|sFc0eYwmx6%TRod*Bf+UZv$p|Dth4vE)SUvbh*6{$Wlmb=ExNS z+Yn1Ow0!FF6Q}h@!0Qiu_f%QevrY>x99DmngUF893K-=u7wQr?o*$Xn*ER0OV-+^NtRa zE{$*PP}x!L$!C~oann$vWF-=vvw|Idl;b=-kv~BTj(=j9ld3 z+O{=ba+_i!0SxfQ1EJ^0CT5o9{VsYjlo5{vgOWml&^7$ubZJAbe}2wcBe(h`sRplE z>2DD5$P2#U7K%>p$yBk2)^fLNwYb{uo! zL?U+iAG5k9sO9hvkCUvjCg5Ajh<-6nRRGf_*=k;RR|A@zj>Dpk6h8$PBtoO}k~_gI z2LajiqD8v|rc_QCukXQ?qZ}yKwg33IE51Ldk&DsDnJlv&6fsQ&lH&cMCPL==h${Ig z1>z?7O*@R>n+mNRA~)ef`tB1wlEv#9h}byBTm~%M_AoJ7z2w%K>!DJkNi7{C@av!c zw6FEMsqG4V?Ais=d!ifBqGYuK zS8WbXGvJAT$sf$?<^28w@il7wxjK4NsO3_Op$K5AA|EyyJ1Kn-{f1XO*xDWdDt?K4ruIsVu9b;bk=fU!0ns+Lr}Mn2a2Xxfe+M#dT!?l@!Q-zgbzp?Ct0q z7ejC$BZEd>l>O!oP|!^+{AM|jnOgr`KSJN?kP$=sGX*2UG9^#qu0nts*6o=0&|LqI zLJ%ab5Zx}^J=@N4{A7s!yA6vDK7D$W{vjmJ{95+Dpj$%Vaat*XPV}wW;Hwr3&&ebt zJ>V>&Z@eFXPLcCP%e{Q68N&I{QS|*Rx0c<1WBJw#&LEXSG+moFn>`*=$-Et91dGG1 zs)*wGpPas-7-i}_wC@c+k}<9GbZ@{j%Fn6d#MtOCPFk7=507fTj>qmPF#cv$kpBD* zzc@&4cItx`stco#AZ>lH@SD*`{m*9ze{?e`GQDRN-6w6u0{Yl$+miEvjpUcx<0y)td_B z*R!>5UPW1>*b6ux`IXk5S&~p|0+l2<*#)0nwzBMXL}4E?0^t|k-)HE~r;W8Np%2O3 zb=B>MtOhPqVM+V8$(+uu{eymGDa4DX`$|a?_;n6bpVn{|C%j6;+o1*~e5NW6Y0f-uf+GIam zO|ErN_srnq;|c{^j(14{&WL&f7$7wYSR|-NP7DGos>rG<%Kt^(EkG;B)`fz3zP7!> zyU@hx<6KE{i&N~y{-x9xh^QzyOe&xLZ)B)5^v8quW#f!nL?ug@@pfrJ7?n6R)+XbN zi1Qsp{cm)VSOA(d@jm*xYqjsrCJU{$zH~864^l}vSF$T@N%ZHA*1DxO2Re>)+4V8S zwk2eiM;&?DpHe5#uH|xh*m1dkx;@|tw~|lvM)cw<4%$Nok=k(3lZjlst1K|9Tlr4| z6VP^*z8)X&8?}CNnWt!ngN!@@UIWYc!#$TTH$t_N#L@f!WWb_7p5up^|5-WOU zZ(}h*Bo1z0761&DT+IcS-;_Hd*Wa8MO8}%%ecE$SctP=B^)c`~u;P+WqBhnCDJvsV z!5U}dCYENt=3oYv8iOg)kwN|4Y|6mof_B#6Vt}smBplwMv0wN)6|{>5pVa=p7C?|P zb*p`n*zRE@k}#&ir3Yw20tkb>ol#O+6RZdA;4ORTO2tv09vFBIwf#UgRWVN2atWU) zymaun`YF>-__S{qaDdf?F@f`1XHB+}%)#o$zJoXJx-+lF>H@G+k{L%m4_-$4u+UEa&<5h>Q<-yJLF_ z!5H1VoS8$q9W)$|*rJ#NT^8|;>Y6$9;(xY;j*k6x^HP_vg%41VTW7%_AbB-QlPRO% znOaeGU7ercQoF#M;^9!Ip442*i0tZOe3w8C2S$Fnz6Tn}?6qVgUI zIq%VN<#2j;k_0zD3Lb5I{XQAYO!Bv)6aI`cTi_`u-t)CK*g#0s`Q(3?cbI1$ptFaQ|vVgEAA+H;;kTGl1 z+x8p9gC&>F;zgfqBI!smbP~f-LnXZWx8#xDTL)F}7`Pnc;;EWQ{7R0A z!;LA%bdO5HwU9Uc6LPe>Cxe z000k5$q4yDQM}Kt>`12leT$ucPyuD1)X`^7>PmbEn?L<3E~icN@{Sr27^Yj=F*Gmn zD~>@Y17>30+F)wFXb!Q!iE?NBGl5v=mkN`WcuH2AVocmI=3~8de{4Y3Ri1PHwYvu$ z3!f_j2a_gn|3UAX3pSly?gX`;izUsLDXYcqfbi@kWrOV05az-S+Q55L)2!cK>kWJB z88aZkffzq|h7E6thq1ahx7}M8wwT|Kzyvhxp&PQhfma*5%>8OE?)K==N3PVJ=;HvE zYNzq}lJHE7USq`3HM#QJwQx?a@qRQ7&_XrkCLglfKFD;fVZ97# z5W6@UkiltkAfZg+_jnw=O>Xb|F*F>}M(s+d1!#jY!ozLS5GEaf{1S!{A#bL_LSur# z%%q+Hu1P%RKJ_Gs|KYE4di~GOvMh}S9t2e(in^6b1v=jOh-9qN;ma7Gt`+#ZvW%4d z%`zx@?p~)0cOmO6cM}w(%*EXj%RFp)?APRac}I}{!Q$Y^R<8_68N;A&kY@?~BqVw% z%1Y6Q0%X&Pc7bA$f%JE}FL&acp&dlJI#{k!sX2?gTB-6BCsJ<^j69xGEwumrv_iNr zkflg?2!98mX)9p`)`4R=oz4w%0Jzbu|7jxlj1Orwz$tyQ-9#Xw`C1MvXr zA8j}iPA*3hM#s)mcmTV_>D_M=4Jd%AM5gdpid3$b8V@LS@d06

et(g3Ke^uc$be zuRLiVFi18pHGbnI(F8EY__MSkExg}v`5lT6lQbVz#bDG{y~>w^L{xxzJE(d|SWd}z_V0RE zO#q%SDS9qrPJR2&m1UG7)I^D)Yp&M3&FpL7dx6z)oB#&>NmuRE4s)VJrj2NocPRri zSg=peqt=KpZNZ9Z9^T{b(;Sug{<<64jn<2GSs6YP+N_!%l+@JJA7kv3mRWc245BRX ziNE1v)E4o|=pIVN{tr!O8P!JHcI`=UcXxL$ZYi!si(7Gr0)^rM3KVxKuEpIQZi*Le zi(7Fh?i#*4-&*gV{LIRl%v^KIvG-0Jn*ElEm1&+om!O~Onkvka?{5|_@%Ys%6jx@x zkL}e<0oa3EiVE^WifkA(?-(QJEU*4E& zj1|eD|FkY-E$nM>qC>mx#;2ZWA`Xz>Ak8@CoaJ>0B0ZW=mZJcB!u`De)E`WasyU{A zP;%~}J$+gHEa8YRs_hz2EaW)3jMEbDvah=-kFPzOlB@NlJNgY6v;4ad=4&=8B#>er z{s>K+AG>ti{QAb**AU0~@%zg-G)CY{_b2CHesDa^iD|YMm9Z}~+-|%Av@9;5H?EM@ z$@ibPk%ER>C$rmNnXCW((2WkePM`OUfN1>368aTnWSbS7v!u^w!1ALlE@&?I#(w1J zgwIwS+|S=1=-*dbX_JKr4;5zz0n4Ad8Qvu7oItNSut~7{z4{s-7d+|Tkdt!9zKKH3c3_%Cwdicg2 zIohB2-F!Bj8KeE$l*C~XM^yaA`F+JibaTkxfTDIy5+l3bdzw3y%>%T%B#G(GS7`dL7xC#9nnCk^sHAs z$1iMXp~n(Yt*mi+)j@I-&-EG($40VA#AKl^s$3`OX$!sDVnQydU$&GF{etYW z)i0Y!!UP$;_s{ZdzsaeN5ul4ke6M1vz|0H6zP8~(d>KOC;h*$`DRMfh|RTl}}B zoHJu!MO!=FVvrd0ZJ5;ybZIf+OJPlPY4vwV^<1Dpss!cPK;$bEgf6B?R-^-+w+8nM zZpMyN+(ybDUOpymAqkoO{`^vHvcMgKt;$Y0Np|Xs}~GuMTHWj6{PI(HoR5G+mTu> zCzSi8XAVO*d>o&!2GsqNf4&+M3AW_uw>121xzQ56BIX8WW{DiryqL8c&rsS_cNobF z3nh4CN14Jyk4HqPfIHlV6~dg$Gc@4Y9=X})tc?qRL(nfTxqpCusYYAjvF_mXmnyB( z|CA?d_1}7Rs|@}n(WID2AF860%^3o#6mh4|f`oWI#B~>cZn39^yE@zD+Ta0Y1H5>6 z{#n*|jxMssccWAO!SQCxqkWdM4@qHvYQDCB=z;pS81G&bbLF+!6D&cP|6Q#6_QuCn zvqDUOQvC88GLZ7zYYPr-#7&r5s5Hh*D1DE#u?NB}2Z zha5_)2ta2_4u16-I{MF375!x^MuUJc*YNO*ljw8Zla_9Hv^)z*DWnft;eD#`$}A( ztGT=~gJ~6(a}_;NpQ)K8cUf?#ZU5q;#NS=pL;ZWqP}6OT6A7^8T;za+do?VNqHW5U zN6%+P_WHCT;^adXYA6jhernNIIU)cmI$iIK+?hfB)^XqN9-irjSn%J8QRTmP zFkZ9;1C_9rl98QJh6JUL|AZXm)9`ofQYi!@Q2nv&nlm@NWn8+KFX30geho-q3MHNy`QSQw%za% z#vrzaW$ZM~S&o@s-*WL6jrM;TojvNe*=y2u z06>Dt5Oe<-{5Rq+YP*W5X(OyfSk-ybPmZmaU1sdN6ykKL8SCC9-KtI@nEyC=b6og!Z-dwzN_3#gZ;4?N`Ixw$3 zoUt761VhKC4Sz6n($dQAdf`$jA<#8`smFCn`k&?ppZ`Yf27o$Sk@Bu_6U3ldjAf|5C7@-Jo$jR_Opst#on!VHk_&;OU4)tpXMUlo z)PTDH!PV29E4D8k0wB-y9ZEKjDL*;KdSVRE(`ZBve#C`MH@8-(t9+}{v#rk7HoCgq zX96-qDxAI)s`VxDl1~nS0P830C7+GylH8hE5cnq?jo|=8uWOveUK6~C`YB@pRuU7N zd5T>T@k?Pzu-a1Aa);3eyVk+?KhV!XiFnDFb0E?36+&4p!(N9JP88BnpHSHua`}`Z z$|IK$x(Z(V@aHkV{P;n+fwIm-rw!RFi`HbimT~r|qHoL>D4uhmVxlvOqzu=WJUQfP z){nGn0YB>dzp>XRxgC{gz&8_AO-`p~+s?>j|$=+Wk!zd zu%z+1?{c{!FT}n@)8bWl&<=duo#qh}>dM#4oca8N+^ zrPsr)@8-qW?bISRmvJsXdB$qIo#TpD8cmkvQbVN29tf?$95Go))TnG(szx?Dc5pYd zl0wD812V(a+~omH&Vn)IW`H>NG=;>qOWo)SExi8yibZ2ijo&do;Qsd=oN*}HvkVGy zh5wnm)ir*ePb-C*IJy2{x!wT(F6WL1+nopf3Eah(Sh+?zBRr?!=A}JkW_zbkuFk5v z6n6mRhDZ|`i0;OsC9>N>dQlG@i60N?g3026Z;%x-_^PH2=f+BNw>UZL4B^Zv>?E)IT5fy38%ZnaYu-`5x%QVv3HK;nrCz|5i$dmo$Wb1W9Zn?8 z!GUgNSXHZYy0fwPn|329& zWS`@AM5sH58cyW=nMOI)RQC1x(~9y{%jE9MtlqF8NI!+gIOp9a`BNV z2>!JE@~Ch6i3twOzE`n%fjM?v`m%`V$y$sqLbVZF4=J3R4u2h*_wuyR$yqTTfsVWW zQKL~w)4>|>9MB512*W_pKQU28UrQz(zih5SXYf;9G?7JE{qF)4Ay^>VoDly02UH-- z2AWD5NmPk#iK1$!{x|N;f8Ly97d*zy96X)6#QC1sS@?Y6{dtIq{Y)LW*0QJ;u2I5&oF3fRt+zXNBOs{g=Cx{D!8rZIF6(NkCiZ7MBV-!LeVU@=ih78rxek#x=y<=re_p5_s`(dnIQa^6W{ zBeKped(mEb<@Pw+t!8mzT!vpPNFA{+E3TgunyU<0_)B*mdIOEOI%T^ z^w8p#O7;M9p#1tAo)$4Y1MwRi#-T5naD{q`gJ&=i91RXv|FSpc3!>?^$s~dmXRO}D zdO_&1e2sN6n+tXRHev{?ax4?;nA65d7tj3Qkx4eV+YnL--HvNq66Mfh|Hq5kaTrlJ zz%z67_0b})5lL)i=p}MzQ}T`$pO^iEL=FmYgw==w&1wjmSa7BWwBh=|fEbKom+*8- z;dpM0%X3o4yQ>M}vEhY2J(-CN=x5B9_w{#8o8)BF>S3xMr*hwAD`yYW@&A!oojJE4 zC|{Dw%yQN!o#i^$QkK37^#UXE8dagD$B6_!g-tG?!e5V-6t?f;>3y0jp}}eKJ^^ zh8uZRwmd)Kv%%o8naCNzbj|mACJ!8S%i#c=Z-4^Cz;jK9yHf&zkznmQp&fnwyr!CG|)LAK(@nJ77)Hm2%j7?RCZ0XVM zmD?o@p;B!ICk|=S(2`;onX4X7-Y2e`2tmc3b(WqLGU~{7#or?e$*Z1u^ZQ*j+^5+x!kg^^X_KgwE^Y%N0yKmZ~QtjnTednroGlML~ET}1|Bpms8| zVun4o!O|C4C8rQkiZPvqRM{HC8fU)l`S5b(YUcy%#8~j%7S0A_RzMunYAqPv$6;Kw z2O7}H4evvu#)k-snId6U27_$}kyp;P0e7gxGZOISaM@)l=dAicolz3SexW~bl>P^` z>`s_fvN}t14Qk6_-^Fu~0?N0PeEIK^KUPAa8x9+Q%UK+5n|H=<@126 z$KbK8w4=Ft3J_@f)KtpJ?p6XimUe15vBIlfp;{ z8A|&dy42m^&h>jTV&pC|{1=Wf%c_Ks3d?!|3W^^EDcN-?_sz(_@;wsBOsR5A>ZO`8 z=pN5u;FNjFE)*90GU#b>Gdu#9NF#ft?wof?mpf<7hZFkGD|9f2s4Qh z4K&#VI9?x%C)*r8lgEo*m%w=PF_`$EC{tbhekzc0m< z0yg9!{4lpqh&QWTQ9H&q#c|)wyFVST(w@KQ%1r)i0ArKIt4w1`7YsA@Euw`5(><7j zyInKlS<8z(w%^{u2g*Z-n|U&`D96&J2mW0s<@m%yMUf);dP{ZSl26DDqg5!^9Q6YV zPEmjRSp2WZD1gJU)pPnUel3HeBr`n;FG`!|!o9GYql93*%jlw=)Jf$IT5J3PEQo*AZ^Y>P-V}~F=)l5WU{v(qX9)YI%+9$(?Mj}Un z4@IC9AAAA72!Z?)CDZbNU}m=5zvYDZ#mSQ1CfILXG%QFgN2^$pxUtN=fjkQbusqWJUyeL2+uG(=A@` z5r^l~3Lm)fr!>K{owTFoNoFP^aYI8tC|UJ~&FUSRwCGLsY{f&3M z=IA#rvZT|?JhJMa5&JVJBFQIeha>+F*O0U;Ys$nZ9`=Yw7I=+!_`Q&$F{A}4lv(<& z>;$vz?w^qN`ooR+kO@tnL$;D2w{B%<11NZKh3t!tL)WdN;YTj9hZb+8P^SM-MV&aD z!(ki{xu!F3{srOw`NY(TniyXq(-$Zr^Y|V=>x)gEbS%{?J`f~L4_ygWo3z3Q@z3iL6F2(@1&qZ zAhKdxtxv+|38%!x+2B8W${&Pv9nwvBJ1N;hQ*q0Yoo+u8=6=kKB8md<0Vq832LS5# zKUc6KNZb*BaTn)<*zo}Fy|lt%`>j9NnPazb=Dx`a0Nc4&3~wU5fb5WFT6)N(1@3>c zv8b#zGMw1&wfff)9T^HI(8+sP+Wfsgg7D>;evp?a1YO0|DYfqtF1yl$X&GN>(Kg9J z_Qxx0Q?6biesDCsU>)^$HVv9cdqXR6$75J9qusK&4Ui-)tsVME$&e%XBCgmb$k(v3 z!eGpNV)!oD(f3oy18aPQhaH)!inv()sqg*UvEwBMD&;qTESWp53Cr#eJjXIBRg~f_ z^DeU~|LHDk(jIJWFQIt@QlS5d0Vpta5&~FnbRnN)2(c!&zJf8k&qJ@JZZp01o1zDQ zj7B7-2qU^60H5n__8_vYOGE%EduL(c@_IFq*f)55q)R#-V#3ctc%28YZZOGLe|j7TbHSMv`R5C=#_n{SYg4hbC3korg3Ib;t^y zuSNb|rpZX31UZ zj&%+;yp?)Rgpj-!g$bD(c+_JcA%j04d5uV3OZ79`p;BQBTS_!m=_#?V7gaLHLU4se znHt`NVz#m8D2z;(2m$~qa;Uv^D{E3>MjwpoFqBslkVN<`U>x78TWGT#QnB%8M$rE2 z<{=0=OkChm&8^HRjPS=9=trBHBg-#+Ca_?@{etixE~+wI{333FIUeSV8)ZGlf7rQ& z-i&67f%TD;xICH-fL@Jt=cXZG6jHF;DW3J}Ym0kZkR+zFBDN)wYKp#tH!h+xk9|Jb z`EgTc;Xy%73vw@agAX*C@xdql>uljS|ClDYgBRZ+C!)_NGLh9U-HCVnTLD2&Z}1dk zn-gy$#rKYrq?VqlBAIx9bKvc~Fc`dki1^)6_+ckm`!68=c2|~*3X*@Cofogu5{7^ z-&b8z?;(|r>%gBwne_|C;D?vqT-_AUMbig?+m-=p&1f*D)SxhtPi zMOji<8v)>JCeS8`?3}v0q&&%~LP?QTE2m)A{=$t$C0ePSg#@Ka`(*Pc1q@o2s`A$j z)^8iY5)`mPq!r;r!Cqz80^nAEdhj>%&%G$lem{)K840kLu)T8k z@AF&>@x6lwi#Xh_diqPti(nstF$;0}FP&1A#ZJx z-m8xhZn8=UsRKE+EGz7e{_? z3xZxEwlQl*O`0=rl*?PQMI?O{g{~SNXX%eGQ<&F~HjBz3u6CxWlK`6dqY4+XrNIV^ zKW}&x%9=<{yH&TI?o=rZnvjmYk3R@<+qe15#16UC9@-YHolc6wB_R?k+Y3KChMc-2 z%>itlxZD-ed5`bKrnS%4Wu<*cjeqYur14WI=QZS8VpS{Fbn{25mD^Git{&wX_pKH| zJnz0U_M8dc5hgiw3I1{`oXCh`2%sLK&X?zVC`v1zqxJPu~!E;|{V~N_MOno-EzXhW3BGWQ| zPsFZ|WjSp8KNf&rm~7|Gy~JKnHWBC?fKIG@GKRu0)e;yG+mvJHR0>5GRh;FKfYbUg zJTlW&E;6`}Bl2qeAtLZx+iiJkMYi3wGUs5pyuvaa@FYI>O%8_Bg@6F48B8JoS1Tbx z{(*v>QfpH_#x!xT-u3%TZBhKaXnT%zl86zJSDB83$vz%7-A7F8`tfgI%}62{?oTtn zUWLT>70#N#K)l$lz`Z=vK!7Utu>G@iHjPW0G!?+0tNd_>dXpYf`(AYPQZMxMVwKcN`}O@|X?~%7a){P~ixF!|c9Ko*T*M z?K3q&fPq@n*_r-x>pRr;Aq^^hn~>f-y2D}5cfaxc*fd5YeAs;U{Fldc)szJ5pn)F815yMl1KoO5;HS+ZWOL__G*OBz2GhzlJnd3(><-si+%Tg`>y)$#rM8Twng+p5 z0z=A2EmVywdiK;nK(f_m>l#X3>K}b+$yf>?GWIOME)&-S{NNFi;d$?5Tb-1SHi3Kq7AHFAt z5~&^;UN|3yVz$B=wYwf!^!?nJQRb3a2dnDZkR~KRL@@PQe2&Jnc(Cy*dm|Zpv5USP zR2_;+tV0efsGdat1R4^j=Fb3nYT6|GL(2~GuGf%Sd45cY#e1xICctBqJOQw`bM?Ob z#z{JGU#tibm%=%ZCS7_<$}NhU81I1?8Jl%_g^ zLgBxb2voGY%VZS9@Lm!#S|@r@sJ~31!D9eKXCIl>eEooae8NO97VC;xX0* z(^S}UN&Z2<)2NC3pUc3X8I*FK4ecPx6kFa=>7Tq*n*J0yu(!?AoDjh(s+j+{MI2B- z)PVi+?{jX65k&~x`MW@UkrIaOn9r4b=$KI@4bu|E(yE)SPn+The1{gmey&M>qD}y4 zzKiK{HxZZf7AJ+(baemF9Jtzg>B9xVT`~A5#X4Bio)WKc_>sm=^8gM*u|dDcY)KCQ zc+?V7qIipwO0;0k@ljWqSFosGCU@pBw%c!&1DstRFsig*p`G_SKxOacZL-WOHC!ut z_VF3HJVd-j8$h`Y*5{RI(m_g`Xb~zwnCainzbW%b;-?h>2J%bYf~%TcIt=(j>am$R z9juLu7wi%Hv3UuzN&FD;SgmW`#f_zyKXu9}MwAX|3+(8Nkw7gmd?SUyb{D9NP22K2 ztvXU~kY(5`aOuuIxxGq~odRT2_}yqB*54=g-T>|~?HXPN>Ws8s&ixY6!@q8ZD-bk- zfz5umyyV`Ku9+TG_B@L%frV)=U;yPos+$D-j8+OTf4lrefv1a1i_eFdd_@#|nO{;@ zV_FL0=WNhn9P_i@XI4P9uv+2(bQMQ)t$49%k`R1eG%a5O@!zg9hV8j%IX3Bvn&>bQ z_NJIZvB90Ej1*y)iPLDQ3@>rRd$>!UQfogF~wFf2(k#ip_`!n^@8e0Ol3C z(|y;vB8N?-fc6G)8kguoR7hkPrp^M0c%2Ikxc{UCXkQV%zZyw26Ve zK-IqeYC-VE`dHW_^GzsD!>ve;9>vjvtRzC?Km7zHd|IiW%l^+2u_oSM#=r2whAS#6 zX*j@|;CrlfjiDy@NS7N_3qsoPgFV*%87!KIL`oNj3~TYxDQutsDMEyMl8b;-vM$Qr zB4GI~YHYp)_e$?mU=(HotswB(x>BTu0A7X1KG(r2nN>g(Kz$Gwqutf&CqEl#9Or*% zxf4+}%q@9~g!js!0RrR>NfR4aW@HfABwdv2*NURT|6VyKw!t3rmwG|Q3;^UC{lr(Y zOI)>d&bP4F@ZbmbM?0JW3kb`KE#a^t0N>#h-kk;7zl{SxP@@qQM#ab*L4Y;#cBB9| zQ?9At>UHyuDH!IbT(E&6Y_|N6L3C7!rDn)lHix&2^whK4)8T|J6TQn zlhfFD=AZ@E{amSy{a-XtZGI=k`o0C;lP6^uY_F<-@T#~;+y)}$`y(K}oQr1o_w%la zsvzJok%dy84O`4U!_n%GrG=`+C*A3k4u^{c9pokiAP6D+?SJ;S1G&M|fp^LQBA__; zW|0~M;5!rHeE+?^*P!dB*m}kPzlCgb8>$zo-#d=;wn(&=7<0q!SvJp&Xh{jR*nMpj z%lcZhjS=)UA3-*3#;&Lb-NgWr=q+)XZIAqU42$+$*NMF7Z8pmf;dBTi!08?R%<69G z&pv)`t0DJnPQ*~pgnNZTn+8aI*E2J4lZ0ewQn8-Z7Y z?>6}iwvg9T0>JTt4N|?m$1=;R^Wn!Wqh*Ai6eIupIi2!M0yMRbD=H04OIk`I5b z7&|P(O>SX#syv_Zt!Ursi`#+`SuaaeHr%(v?a^fl{uX{1b6H_lJbNPZLoOjgTZY?8 zTbGz^T!R5eLB!t~ZWd(C*vKrI*%BHgv*Kv+Vx%gcchxjW_-fUrfYoQhsrLnSN9sdR2B z9WejSMq#*NJOlp4tqSW;>y8X?MT{@*STb#2sgaR2Mpi(aWlhs@8{uk;$J)CQV|JeA zc@hvn<&Q`aNqoIykECI@@`9subgqR)uS+#>rhG0i>HE89X)dx9D++!(P-gdRM0Td$ z)W17i@xudx%8~{2?`o^C(Pzk%pNjb-JoZcLIu*p+x)qQiAqz=h&uKX+Vv>TT+wclK z8-y0B*bf}ka%0vc!}mkt!K@)Kk z;CQ#JgkBI}$_WnGpeIc(G?!w0=?kR38HDse`UVOry=z6)0`it5;0IE6@mXM-Ct?mZ zA5i2*9*q1U1g_ZS+3&rgvQMCxT?4;=m&ew6`95~L}0 z(~-xZoeHgqGsQN^AmQw_#i4LD|0Q^*={H@+06Y>f;{Wn#9FMx}@1TX}s?Y4c?&{1f z;AXup1`GopT+~CIV!Yv)vpfy~%uu2<3F~dXAL%W2`&#E~ZYNM&-iwuYV5!jz& zco&l8w7nsZ!rq-Y@JQnU)RiPeoj$6M2%^04ifEhd*rj(MH^jYNR8~b)1WC+-zK3+2 z(Z%S}dK; z%g8y4OyYh9DYj|-`c@}lLHw{7zUxL$KoAFrm@C0lO=hrOHoYe zyIMRewBWg!$p+3gr##fo!9zM_=um45sd&x09A8|shBMY>;Sk5W6#0!&rJ}`X36%%l zu9js0I+Y*H-F{xo&rf-&1l95LClDXQ20kMzGc36q;1?2iTu8q*P#&QcXNY*i%BgNW zwaKt7I9z6qQK2{jJkZMw)}x5x5slGWHyr};8Z7jdfslX`z+yGtADMXnhr;qxsxY`o z8vF;S;oD=-L{y`@3!EI0VlL}{Ro1i~zHBzRRf$EJ3V++iYxwMbt~r$<K$Z+7gs@SGu0LDqe- zZ}CM4;>OZwI_3S+Az&*00bg&fqF4Q#kZ%Zp?Ox^yCPV#+L$j>30PQqv<&{YKGf53# z93=8iFuD0pHB8gq!ZD2?h!xb@vcJ1~vO(C6=o~Rl3HKv}HvBM{^?B6Q_;}bj^a+5% z(j>#w_7ggo>bj2vJhm9$R8rbNTdd9Pf_*3Z#971ds*(B=Ur>>FXXyS#-*qec(Fdh5 z=P_O?T3Ft-MS4Hc|Dsv84Rc)G_Du+v$75-w;L~cc6(EGg^Km}5H=mF@*eVScm<$4a|U?GM|A2lJc zCIw(X=Uipd_=B!Kr*sdcplBdrZuF0n^{>bc7K9ljC5r-7=lj?rGfqWuQ@47v?#ATl zV-%E1=sLhK_jKGehXKsTUt??$E?=a)`RX&r3K(?obNBf^#zY4g&}#gb8kt2HcC)!n zYC)O>TOTMo8~sZVF6g9p@S{O&9B$WBB}AeH0Kv6=f9rPo2k_vP_`-4JN2WcL3rbAuw}WOb(oD9n5>Mvr8Upx9GrC zmS)imTf)>ps%iD|p+K@jt_gbG)fCwUG$l+o#5$cbDb3Bl36W=&lOY5ZE^#Z54Y63F zXCVDWYU8D9aOD4|1cFZ@JELb1`}-h9DQ-Bje`i$2cmQA6GBwq>HHvz>k~|SCa@YON z_kQ-Zf*u$`0|RM=kotW>5%k_Q|9FxqaktWhGXwF@dw8F9{updm^2$~4;30M&Vg+-o zm`R1}RPSNFCykpjNxb^7rXsjH^=~aWYQ`$?s{*Y#>X?CXw#oH3cs^Dl z@a>8pG4*1S?>n)P2Zhel8OG=R2@9g=X-S5zpl2E-B#Vpzek*#Dboz&Xo68s~m-XDqjcGeO*6ewa2`?#UH_|om6C^ao1com9>a$K; zTw@PILYa0C#xQb0(a1yWqOl1qd$(Cq?YJ>eNyKQGyplE0iTm&brmoSN;@< z`Q>r;*Dp(bBfebn?qRuPJB!su>=_|6vm|aAh*?N!Mi`#Px{)l_*Z9nj5>wNnpb)BR zR#w1`f`%-uzxJgOXPLEuvRMm2C;c!>uL+ocijT&3WYL45Z#GWoPiUMy1v3?}(~WiU z(^S5NZI^KZ^-lXz7m}~f>;ExjxN717>G*1h0PT2*Uwik>-K$xi|1E1`GjoQm@vm|T-w%~>Hre(vR)^F2_Pq*K7CpY= zQ#}scF*f;RSQ4Xz=)(C4ms!yvBRFn&l;G90pTBZ_(c1@ma@f0{^({|tg2x_qjVChJ zTWJ(?o=4yH=xeK{N6|Qd$VgLNa0=DW@_Ou9GE+s+g4j1q44E|5?fNH?)qfEGepGAP z#~DyXFZC&=mEq|+6*Kf3bkZVT1am`atl!?V=?VUBkcJO4ZL&!e+2Il;EvWcuVt>@& zT(WZiL&`WRYS?epo5@ycErFn^(U`}MV71!rTZxXRygoeuYDi1Wv|P8pI?SDkG(QwBl$IrI|mh93Aq>Pr=zbk}zmJYGk$6z1l_CHQ4i-BaH{jPldQ zRMgx7;>YZqIJ3T`{VuLv{4yLjrf_$2_IdV+^d01J{EGz^(l-G8tqp|tUTbH5qdgn(703MayEO_ zkbZE2>y^Lv0?jIw8Sg|=MdEUp^ehK=23B1#18OoAzYI$_VTr>hlbZ*M-)OAE6#H0mYXQZ6@^QuY38jTi-PHvEiml zae<5p|5^0~eLr{_)1apM{PNH3ST}nNjS>+Ex{VnQhaGG5G2X4-a6*|!S%m=25>3b$ z!EZ$o0*3c$o5PL)CA0s8pPv0RhJB{f`%HfO?#wI0V$1{Msu3_mLb5isrrgq=D};yP z_d4GfSeNE{r$sO)qie>BUD2uK6zT+feD3;|c6@+79k)rUH? zI0=u%4^5oBR54>jZ2<-f`n4evVyGdsY}1&4d-i^i5_sR*!BQ^h>Bdc|z)e%EfvT*; ze~nOH62K)*zYE$Fdx#8v{=)Bdj8b5yBxoB7`-TMkR(lxhxnl{yk-R>hs;cX-6qrEW zv@;PHF7Ri2VN(GG{|#PcHiyjxwmW9z<#>;{4a+)+F68fA`GAl9y6!w2Nq41<{MUv^ zd&CE>+}{npHMg#BGFEtd4u_McZKdJ2ly{CO9Z$fv)4`WC{SzKCOZJ%PnJwM;#Oy(z z_{f#}PO`E+@`B-wzLhKAyE9t@3gci;-)BLf{=E$uE+kwd?dPAYm(J?gZHv-%13bho z<8N5=3uGf(K6JMoansJV(x+T%p7ghvqnjEHHSdRsdD14NuNCLQZV!&LA(vh?#UHUD zFIedgD?iWdbP9xU1vMleN@69>>%AH#EJ4e_O{(@k0LTy4l)54brVV~D->9J0K!6$% z>l)*8AZh*_(0x5nl$JW`dWp}M3c8xC>v7d;a}_+?s#De94|na`MtiuL4j{sVa_>H; z;f60=d|Qox!)5bIlN+9wJDDr=6H*c9(nqGk!RHKfVXg)$^2pQXnZAP&B*(B!6x{$ud;!1ZZg!A< z=)N1+5I;ITqQvM?n!cqvze zH%9kmJb^0fuso**U%SV=$iN>bZqtt)H^sBFY8R0{YcIwdJ$F*qH%_%3mjtTm&7G&P zo8Rn51$cQ!vx>Ll{_9t501%@B^iOva(u^Vi+h&m~)|I#Za7$ue$%yCI%4{=+B&Rx^ zXnn>j){|3a1gK-+NJ|*csP>zl#oJOIhMZ4EMYPmJ0BR20u$pqqLxZ9#y0EUVg& zAWTpq0ZH+$$HfwhR_%j8R8_gUFJbQFTqOufZzdwV41@HodOvN9-3YrNF6oHcy#H<< z=$)>JZy32Xqp1?){)CfvzUKQBXx@IiTqV_IIb^AhO)9)KRbdYEG0z{}n>-srG%o5j z5k{{JD4jah*k(3pX%2aj)hUfBm!u|Fuo`FwAiy{8_yY;+9v%fm@}9 zsk9sokyq_^^~0aY-?odL+5bt=9yB8Qwwaw2WIq^)lt7&v2#3=g%E%+N*Na5wFbN2< z1ojvF)VLQ7C0Exqnl^&rSQACJt#tm~3*2o#Yh@&M(?MYKo z%u94p|MyTv=P$9k%$D&yX09SFv10%Pp#?Zddv8j_PCEaPT6SB*#Zs4(sd#R{M@V7) z!VeL$7K_G?g-cX(Kav^&#dGnauIyGZ>AxbxboQ8lg045ji^r2VpA?*PwrU1FHv4O4 z0L>MMOIVE@1fH-5Gizw`X}|9|3w~AXeLD3&Pj9JOfIX$F3LRXjM*iZBn-(Adx^G*8 z8CHC5m7=+D3=!h9n`mt2ly4+%sx|s;DU{m&Ef@lFEiens$G0%!R~U|x#kPWS!IC^E zJm>fkrGulCV4f2wa8r*Y{j|WovsC|dHhjlJPL{OacT0qJeJ8t3E?BdjR4lggZT%8} zZijb?0>TJ~xBhJp`+@?+9%-Gru_vjy|#)|D&$F}xXK zJ}k+v+72D}@G76HkHspNieD*B`<~S%!?6UB`Ln)Fwv*`yb){Wu2tbQH|5~J`t#Egm zmWv}z-~0FNMd=75c#DRDAL+wpT0j>Ux)e5$>N@K9%b5*@cy{M-OErBg3Bu&Qh!xuLOBK5Y^d-xm_3vhBC=%Q77VBDeC?o3vdK% zTG70IUwtVaIVYw2EgtdSU>N~u#d8+`KH272UKUAMy&+EG3ce&04I?g)Do_x!CIV%` z!=<8}#Xj=EC4)uWa=u_q$`#**lNlD5=BBQTfAkuqo%Jz3g(JE z13x+UkU$V>&QI*jd*WidjW|ID3VL;EZx>YuQ}-i=bv4y&EYpNc*=q{wAY(TCFNdpG z+piTXu3?;4Y)U0?os`V!1`aon!`T6d3aK+l4|waM0K1c)aeJ^E0sasRv-QW@e>>)V zj|0kZ*Plwi^j8l2#B>0bA~=U=Z6 z+?KPbs=WCh=)QU^u#!UBXO?GdS*v{+q0sh8}XKdxYH5Yw~H zRKq`8>J~RkF&UG+%ji!Sz*#4Duy#H+29J+iJ?XHj$^$k@)2KZbXa8)fY+~|T| z3s>{+a7+D@5CM7N^=u*Epw1qXw*5M*Cy)9#%ELB-xMyu^cJlxim!XKkY<)%0VVvY;kXtey)+=+)8@u?LD1?s=G55MR9 zav}YBmX=@>Of(9-4wFra!j#Of@gxsln&R9!Ac*jS@(v|AJvGf7aNG zyy%X|;0~X^17AJYKFp2JA=i_XG@sX<3lS zp*h3>lhN%E@XW~#5`o7*AK2UV`-n6Z=Nj+9;*X5|Tbu4Gt*(I&Wmd(-!AC0{p07g? zhBM=F5>oOu5W^P zq9y;?N|fkdYEXZh`)D%E@(+^d6?YFnjs$d)MV*pO)$2Kl@dRo}Gs)t>OpFwe@uoZ` zMB%%21mKeb`~%*k34>jM(b9K`CH>ZjaK|rD-RmODE}~ck_FM!%x&_9D29wHU@`OD; zk=A0WO#)@Qsr@h^`6-F?T#=Ty0WvM>UjemZ?eg+~^T;fnT|K6i5NyU@c$Y+3;Hp7gZ*^f^-z$5`Vs0){~CB3dvTJ-q4wSLsZtsfIzMl;)hd(5A`t$$rH^UQ^F z%<3NGrv1;G0AY>0KT}q<#o*X4gTuiQxlD5VJ6{h~6*o$#(6eola( zS#ZtX_9|ro38=iQd$HyXQdzo_LrRPAI^*oW+KeDM?WS}?_KNclIse`xv(_M`8`3wb zzp9206Z7BuKUzQWoRNrrL_CJk{YfSZ@hX)XT@%YbzMn~qsxT^DETJIe1Q!Kiy!<`oO%aiyZ1Xto}uIdd5SgNE|+QhXr14-Su+( z{mnWWTwasAH%K;4UdHkF;KZ~B>E_KwvX7`6B0PhvjG^Ddx>;FM{G88Ad+jnGKCGDK z?tt&sm3^BGJ`xEnB6ofcqJBYq{ynbvD8QSgE2*(ez_#?VSZ8w{ZpQ?a(a7ZG7tq?b zxKj(8c+>79KRUBqJqhy8!WI;V2EFP69)F!B_C2|s=>r93g9LUS>DrQ#cE}MKJpW|Y z`gl}<&6S4YGNz1^r<3n-*;5I_i$-IsAureLaAQG4$X=w6GzZPo&ITRhTKE&DKC7NO zCZHx6DWS1%bJ#kGZ8~%slB@mcvSRnXeS;LCYl zDwu2>yNm8d{C@LgrZI?eN19fvwb|?NR%Gc8KKvbv&%E>f-_Fcsb{7XQP-)qp)Lx0^5={K}svTMEHgpU;={B(FP72PQp0PX*)=`H-C`o8bs zJ41JeNQrby_Y9rVA&nrNN`o*12q=w&AT@}jAYIbZAT22^NQ-n0GrxI%zRxq~ADH_( zcb|LK+H0@%)=y1XjN8%{8DR)C#C>kQ)%1&T$Qi)%GnB4r?Pb+Ye(>f5Pl;|>ko+@`f{^*OusC-X-lmzx=*md9U zbaNC%Yfs4jTUEP|=6L88<)1sZZ(>QlegJeO-S_QkQ^77`C%;3dp@?(E#DnVA#;4hZ zfi{RkQif5ksk-_iA9KdR_-#9#Y-3k$kd=!oA5EZ71ttZn5JM)k-)AOTqW*q+sHBwt z2^*MAe%Z-b?B?a1*~LmVOLN;KDl)(y0L}CSp=A?+%F&of)lam zhrTZtD6Qwyx#tA3D_WmWciit5i4pdD>!}Zaj6B7Qx6Kbq7QpczSrjQz@O>Et?#5r~ zE{L+jZ|E9i4x_){KXpJ9e8B@oW__56W2P5%0u;Tyf7M|o@a_9!ZcbdzXK{W_c+bXy z{=}2h%kd6I%Aa3l_-&qOrxXp{DRWBehtav6JD5wCmy-{ORGRhqC+Uj{(Uow9D{u(B z89UNMjC1oo^GpnxYpvQb`mxEa`N-8NcA&}gCH#sd!9+CDAOuxmoAy{vCqf4-6z}%H z&4;+VPa&^8Cmo*N?Uw>yaCsi)7%fW}U6B7I{_=jV*)@+26j8?P0%~JM2i0J*PwQGY zwwqnXijvbvc)~z}1?oJG&0~&P(x>v(WCt^PCLmyryz)?h(U0kqrB`btDdaWtlbPDZ z9?*7xI1WW}#@)XtN4{Sc^0H&O(ni!i!=ZPzIli4|m+DvE^<&7#R$ipA$s}HWKo3q7)>+%sK6x3s}-x4WAF+1KI=e?GD$&!$y5qLNOP?1Mb@yfZoueIFZI@98EA?;gd2zz2fgFYt#;c&(2MMcN+ zlVGjF0wv++dmXQ8G|u6xrz;R-zDqyh>2v#M&pAiUr7>#b$KL&AN2pU?9uQ_l3}{2~ zfXNo?s{9I-J3e=tzP-Y*4@kH@vk>F&qVqov$rObH@30;+3oiVIzn|Dnxs6REHTt(i zL7qVq4#UPTYs5aU3~!WE<^i~$x~m1tG!i>`1yuV@|kjQ*7HIfT>*UD5&4 zN24H1KYyuD!Sw*R6BCm1voS4?;EP8sD3h8(bL>O?OKUr1=uX_|5FJ6<yrW`xJ*t-spU8I)GEM!km?a3lEn0O&t@$i&z^h)4BxF^?kTU#h!Hvvuf@(OaTww>RVi zs>;f}6zQ)Hjt=To8sg(J$#n5v@8pp&uLR_HlD+yiu(Rwa-fs%;pv~8HH|IFlBS=e zs3co#x>x9^YJuW)r4mH>FPWsEfb~Fp(O?q0&7Ziyez+g_0^Mh4DjLQXK#fJ zjzI;VD2C*TaSp{s&&pm#%>D5UFReC0eYAuhm|sPl?9b+SvsrW`EcFERmk29hY&bV@ zBGBq!AWE?;6vaU1qj_;u%QYX2W+*j7i4XwnK$Z@mAL8AinuE^kA?i!-u%{}qXs?nX z@9y)6x%U=}Wo^m-MzF(w8v%>~{baT77OOK>@Q!1uxsvmRic4Kvc-b}|)z8i9OM#KLK zkDf7~3HuA$T?j$955GCF&4+olGzb>D|jJ54-o@(!r%58I>{)ePrCn%4W~UG7Z)*0B8A8`Q@cmqpqsQNwYd& zC4zmLxOUSx`a9U%omSPbXB)l`Y)!8{>~~hs^dAC;k@Q7fA2C{hK$eK}))rxO*YyOR z%ed7)Dq%_O48!MN3$bv<&?-^?1IX z3mVV|yV~`_CQJ1S9a*Az9^jdSMThBm@awtC?{M;$IPH9dd*-}~u7?KQN65c@ebuydWgPdP3%Mw22&asY-nKtH)48JoW%d_m~c3n1gydE zr1ZcB!VZ)<({nNbAN`rnqUL3FkN6TzsI;pIpdNn2P@eT>pzuZNZS#!R%K1-GN#Jt= zWQUb;W9WLuUo-@F~l$1D1#n8vL@N++n$t-;~HP(x#7};gE$WhRQg@&tL%5t zxK`@T=nu4x37}N(w@E_8-m9F${#8o*eyirqa5le?6((P1-Xp8_5AbVgQBcAtRcPct z@HJ3gQ@b3?>lTaieF{Ry2>`{e(PNj-Ka%*=N9q5dMetj3KMDuQ$nMTK=c#_NRv=v5 z2%~tC#jB~GwhA7buND`I1StGAn`L<(YU<-=pyyV7AKoa&JjbEju65vo0|1)?{OtKt zemP1Yt1@Lk){X$!&qexUY5cyE480ezguI~bzWy_~a8(`@rT-D97^mF*_p%QrKqf#& zsogLvBSIukeqP(V-M0uIDf&z3gBI4El+wQKr#8OHn9jj3V8DtkXg1ED>~d^Z_9y>C zM9pg+9vCtTj&*yVC^W4~XM-2X%8j3nN4rJfmJ-dTqWW(vJVF8wAje&UdBrlh%=gE; z;*Z$sqY39VG0~EvU1GG|puU}cM2lK40(CF?jLU}8R^}`3YYqm?J!oXe@@W?LXCmk~ zAR%*i77yNde2|+|uK7^WzGwnEl8=`Cd@Ltr}Wph_3+cYL=knJg0AQOQ z{Ue~1Bs0@PR||?+cB84lAj?}fceW7SM|6hPl|Rm%YdOn0R@8*L0=R-ZJ|#qi>|O*p zvZiS6a5E(Ce5&1j6c%Q!mq}&S|HbSpqwB;UO6{HTzl$Fw{}Phbb-ptM5O%Z%tfV_n zPP#F{jXGT|*U1221AnY?unbGeU_^wJydHOkFSrl8Bq+BW|67A2j9E?atu0 z`vCBrv}jeGoU%3p*EOeV=xU-J#ak70t@vu-`_>28Y-Tqdjbx0YiD<$9Hp2mhVsZ*(Or2dPsx0Tx13$7?3izE- zJDJ2s_1miQ3lWs^&so;!Q|nOJ`)^572{Cfqs*gs5fd>_tNf_s-C$ku3wF-55r;0*z2Dta} z`J4a@u^g8ieHsX#da_n9-Jsjc+Am|o&GpYZz4<-n2{6y#LO@-&MIDx<@p)*k!J)a} zGt@H1Ciq_Pw7Kk;P#Nyb5v%>kvq2Gjjnewo)WXTDklQ8L7cTS^r05Z_Q1kpnxJYV{ z{}*ZbDu9W$!MYYz!wmc*v9AGqes>{nZ1*boyp%hO@#aMu;T3*;kdX-Z9u|;wE4k-4 zQ#v6RKiAk`*mvBE;3)~>+vLQCz;9CK#ho+qAeP|Vt-dPxR&1P@+r5h~)$D;Us2Rx^ zk0u(jmRh)x*h`ND0RMNkHvAl{n1#>)+&vf!LIW>n%3fqz_bN?$irL)gv>SduWN0 zqei5|azWkQ-Q-%M&zuAN{JJ06qJ*&lm%iJL*e_lmr{=GocCqk8Ccmn<{C z_MIfQ8YUGxZ9pqO(MYbk>u zPgv~&xE_IU0Y+nj;D@P~S!PCJ;cX2T*mIS6asY5tjvw}KH;Nke_ko)-1M8rd4K70Qx!p zLBD$= zFrGsyko%3y_}wqIwi<$4_wJxe2N2W|^~O#(D`D`r?D^%Tpy+-%X1toFyXp1fQrAT8 zV&L0R96bM29B85qHLo_0USBrYd{sOS&qz^ePSI8lsT6XbF`KSYH(?(9tWf{wdi}}o zTr~^^Z7h7U(?c4ms5T1$3R%lwfskV>3o@7gR7jA{;J>!rVKiM790z~WDo6x=^gM?$%oRZFY9R5#w)pJs@FM@SU+!UlXj zTQ4~|4Xq0m*O@w^>`n2Cmj#z@cKfM+Jr8A2JBO5N(wfZ~X=DtHl>9h*`0^1w<-2ue zeEN-DQg_-mv`>l+#_Z%385#Z`NELPM_gk|=@Fcsd#dzzmV!|Jx^6Z_;)fZf}f6) zUnt`A4b+8KQcmQRyPn>!u7@|MMjJC6;Ckz+dn{eV+>TYb^NBT8c8#=(f6;F|X= zKJ|2OAl1wMTvM{%?SCcZ0Z1^K2M>s| zU&Dwq-LmUvZPPSgEwKcddQe>7dn2&O@}CBfB9&=a1UsDqTs}iy+sMV#DJu6ikLS*Y zy^>y9d*zwErWiOIL!#V)knN8Xmvbymf<+tby- zO%KkN_8C<$ZY`7DBR)OswSgSfY%#`7XB|SNHCL^mM@bGdRuYymdqZdj5Kp-Y6A}*+75q<@6Fw{wx}I^YOp(tUI-yL-h4Pd%ftrWTm5=a9xh6T;%|k736tD_9kOV+D1m?LzU$BR#I(@gPz4=b}u|F+c3Zn{V(X7oEN|f?fzDHwzOo4 z8r{Q)EPnrT?eX*&S$~|k`s$YD_>Q?9Z6dRGU0}$$%@y0LIQ)Y02?!?%P*`)a2eMCT z0bDta`}#+x5JCVtbia1hAnH<6A0KlBfBp8-r343|Z)A5URr@Ir1YG0c?tWtroBaQ^ z0C{|ZoDVkvtDzs{wS*PRt*b5A!0+HlA{^GP1STLrtBpd*Q?XR0BD*Ra* z9WhUrX+Pz7-4lfE@wLSxASYoc^HKGi4{?!KV`a5z@oM&K`}?k4{_WMFPXX3xmTZb) zj=%2S-YYCty)m9Hv|*fhlm`TGz&3@_B%Z%v>!;;z(=TSV@gl0f*m!_aEr(q^7yzWj z0u=<1eMw6U9V`5i`#^nr9g+l|f4{u54Fr!Ega$AO+{i%LQ;j&@|od2eVw2 zA^cb}7Rwj>oxxWy{|C29#g|uP@@p$AFS!HIwI`i_#X&&l@Veb~jR&Vc%*-q&XB4Nt z)6UVh20wkGdOkE^(T^gYlIe7@emAdq9*)NX_GyXR1jMMK;#dac){L8h&x-}5h0kh# zC5OJIzW%_rYrUI;M&s#msEPQmz!FF># zuC!DTr(=R`5a&|*4VlAgvG^H7d1F=~_UXarkw3inGc~x*cH5UNz74;1MfdicDg4Gp zpR^rDQ#u0raG+R{do~YwzxxmW)xlwWNp7$Wl7 zT|DL+ydQ|Qqisl8*Ifvs7YaeTuhWNTsYm~i0+opSM8&E1y(FhBWJlLg^{$AG? zO4e(7(`2pnl1$L-*C$|LY3uOqV7c!YgGER$iOOR|w7Nwp6T zzmj4{JPT?$V2G?(<4_cQW9ix-_Uv}0R;sG?&L*zo61UNg)g%3L13qhim&qE*Ao$;l zOaS3#FiKU7xrmmuVv7N+=^xyp8!a|p)RWEF{zYRQ1$OI1Zzp5bzKJ)Xpg+JaiUPp{ z5)!Z_{hFFRu(ru$Lp;@2mYi4iQNNJ_5xb)tojQ_!pKE{Y%M%^Qp+9q>`<1*=g=9z*9ofvE|%BdgryLgKJO*W>Avc zIpTY-FnVw=u>c+U`Z2{n6c_6iu_Jj&ch=~*0#C%49SwN`-0h6RwZMo8F0Pnfr};l( z9i&V^jR~ZBhZOQSj`CaU=qkY?`xtw{((^AV(!kasZg!;K%*_f5Sh#m)EBv?l_3I^M zUX z+$^{-iY#<_QS2oeMQO3^Z#b0HRT!Cp45^p94T&);Z{p_sU=i^oRlAVdD6V}Ndr{#w z#T0p>>qZv+$o)L3UQ|EH5DPFr7qGpk=9P)Nz2&XW{2#Df_{7fYvO;ConK(@%iJ}uw za5EFv{n|}#q|)R_j-2=y>Hk6|Z#Xe23E5O#de(5Vh5V4_z5>88Og8b++d-tZw!58d z>MD2AK4!NR;@`|LgrnO^chz@nl4393W&utV{PGfb+Y@1bto{2~Snq7^rsiUqbT^>cp0+W8v!%(Czw$j6FX(KCBX{blY-nFUj1R z%E7P;?)?I?sBE~04hY*1j}In5U*$UDoBN)o^^DgI>E_NzXxB2`E`JSOFiMLkoQ z5MGkvB>4nm97Dv{S%%$Mn)XvtPG{o&k4#-sC6}HYLCUs1Qcw6K-f$d=C4J^=?w1O4 z;;eWX_cUMdfB3kjUjy-W2Iq;%VLs%z-Docsa?8klLUBG8?Q>TAn&<@NfUW;2+dQZ4 zlq|N0plnXBsrk;v{mmo;E>NTGS82P3PW$>b>5P%osr`xc2mo}lVfjt$!AW;NacFxX zAgf5>d`4Cv#@!L`3lNW?VJJwSW=2lG-m>`FetAx-a-$yd&2?Tha}tW683;VA0})-^ zdo#R^plsfXx}nb3%!dmU8W2M6iL1gkDe4B*6>>Y|ei?cJN~yqKhg?HuCG;7>~X0v$CTi-#gRIR&FeS@}03Dt#r(zG>p%Ub6tz~EF`ILxh z%u}9b5KLX^%0>v@+&qcfG^_d^oX;pKWctoY*RifONDk%Gy>rhi(MrOHnAQinBUf9V zwDTzqGrJt){}6W|;D-qBlNNMs0ffll#bUB4$1bFNG1Xfv@$-*>^&CmxIp5!RbtdbHURu6nMe_Cp313?>5iz&CtEc*^#^PNy&#N8Pyu1 zJupuXWu<>CM(A*#F_W#{E?*zKUC>)4rb-+%P0_Dcl4$^xmz4nE(}sdm1c>12ctGWs znz^%>pE1~%Rw;tl!>)dR%+>MZY>7A(AC2kF>Y0Fn)nliB1u7yp%!m82K^?1&CGjXJ zT^A`pAKLwIe%Wh~3Dh-#)h9Y0x?8kKvz-T;pRx|@R70Pz?67{U(ucnw6OyfXi}LA?VAfF&mK%AfRU$+(@Ar;i0ce^66~<8F*o4TeAIr_*^$GjI&% z7Q$L};A`{Xs8I%h6uAk7OPAB5rW7SRRQ%(~q;lAsXge?nxQYg)2}Q4W`Mk0Dc4=|# zK9@%H6{uRSzBIPrS1l&6ybXJ_P~UFVbdF=!&e>LW%u9{eo>p&<_{Kyt;qn5KBMdxY^W8e`+AARB8qs&%%3+5>8xmQtsbK^ ze*<2n@-#Ijp?11M;w~V1L_m}Y@98Vnzrf_`Gj(P5?zkT_Mit;R^ypWZatBL@J$vq@ zpgpnzdBbmJH_&2^9k-jm zvyi$b@c|2!QFi-k(7(^f#BEHaU6X?<{3vNvNBebpt;x$yE6b zuFTKmhn`m;QI&NAf;J9Ww%WRyT(uc030adAi&kEUF7&{I`jE`^&J&r;y|@n&{b0n3 z0Oh+MZ`IyQ;(|McEDW&d7eOX+%I}osKCwS|-g7>2`)4IkxgqF;ry*n(U*!-hOxO@l z)k~g=gBq{c^=Dkh&({tB@RvvclUy>KKTz{4m)e)j=6Mg7%UzmY2}ttpo`w+yUJ!Sw^g~b&v zP_`M)QDJeGDhiN4O4oOrG;lZF*|p@#Q>|`qV0`JB+nc1N1=Y<}D}#K*1Fi+%X5zgh zMUS&K+dc#8yy6OzhiSQVnsbckt#jH?fT7PxGL(nSm|h6OD8;kJdhysULv z$|G0`e$KKIJaO@J(eqJyP*2ZD!MxvUO48Urkwg$Y>1}ix^*c4R>%_Fdk?4|HR`q#f zfkCv=*N&G*BX+_qKo9Yie?#+;eLll80>HkgcL?XEosHOqd9!Q1yIT7s-n^=m|AM-lXP0XR^jJc1LX882;Vwc5GypcxkRO;e z0|bB1h30)@o|550s(|q# zLGZ`_!LDwnM-ksGDoxzC4_y~N6xIh_u@ufp@U4W;H=YmQJMY}zYmv6(e<*4HTCQR| zxEJ1D$h6C1lR0eOso-uJs+@)Tp`!2{PKKTz^8yB!fSo z<%Nq2)zr&D%!}Do$0%!Ab3_m-tNxqsjiQC-zyPlt$EK|?HCu(P{_bWz4>>IcikX$>A4o7B_{mhOUUVl}lvtJ>!-XyJ02D2`r}jgsi0OsVrJiTqwY5z7p4-@AOp z>Bc=!<|~6K>nSK5psx)2xDnLCI_#$GWo*J!C6!9n(G~;BHjn@Ba(SbLc&}zK=R|55 z&lS0KS!_^z`Qmm_FPS12m1{koGamxNn9!)Je}7n)y*wWpYMGy-VXi9$tj8W4v)od? z=9q72Xh_HrsPSuAkn8o(&P5KEz?y;}Xk>`YPG$UVN1&O3#Moy=LbC84!fqIMkN3qK z>13Qt9T984;#&#HF@-H*H|cYI25cf$x{YjpfYpOl|6KkZe#gbZ=unfZfHV~VT_4Jh@ciGSMRGgiWbqqP>@=BYl9`a`+oZ-Cc|&|2{2 z)!o1`6~E`Z7ccaWZEPIc924|tpkYk(&~Y>xJoj&upG~lWJvP@R{NDwWlsQ$W1orFK zFL^w&#F2V#E39`PdA6Pi0^(}WXLTpujd+NoNRZaKuk36a9$5~1{!;4SAf))B4B1R@)zAZGjl|;W0mHcsPrH8} zTeQ7jZM&o^LrQAs=BO8b`J)9(31sLhrnCE$`7oH}KaCd8BapQRjinL|(EU*!Yo&nl zO%;lkC=a$#ZikS!;(py{v?G1rFPdN!DpJ+4{@oyPJU2pKSmYbwg=bR!%CDR6FNPd* zlP4DhlM+-eb&5ZAMRC|YS_Wh6{TW1Mh(U1u#<#Vyi%}=^!S>I8m>26=XpT%b3=bsLcs%{T#__%;JnHrhDV@Qc#_w@N}4 z0vJ!nSv8Fh&BD5boy>D#{5blEkWGO4r7-^6ww0fOKVJU)<|7ohu#(=;yM*wnZWKwi z02Q@6_VCXYX<=Bv9%4cz+-nQ@qLwC;+ zlh~lg>MgR|NiKnSKTdNwSX;dQ`CjGe&G^rwA{1W3#Lrm*4$Y}|4YZ}}auB;i_2i}> zO|?`YwjUzJA8nsae0XNA+>F)>u_#%rr(RUJ7%&aONf9_SE!=4okhL)9R z`#oe{|81?hI8n_;)CM^{6iU}9F7n=8uT-r%O4ja)Fgy>7e*_D3PyT^4lJXtok~?>h z7IcY*TsJ%K&6}YjY+%710EqEnN_Wy?+%oA`f$k$84o*C+>o^|PQ{)Zza)d81tx-E# zsPb}TMfa~Y?UV!;qh2p_GCt}zwd`1z4K)@554g}U=7tqRXmOtapzoi_>S^)R0uqR4 zLc7{xJoq5E$6bckr8OI7)uOe)kiEhH)(cw5qiwEjXZDCa1sLyWNqeLKc?FH(mov{T zN$qI&#g9MSgGhPo37$cNPn>0JN{T1U*&6N?>i5!(l8Rumr>s2(6LDP#>J0JslIiWW zTgtA%$dLknUAELA60+pwgICSDE3>l;b)AhBsXSP~8-xMW#ZT;2#pvEQaue(hx9w$- z3DXDI)SWyr;M)kQ^OkE?N?eRT$MMW|S9^p_*!I)wh{`R$Yo2T{OaTs`*#_a;xvA&V zSIS*@^2**22P~Qk4aHW5FWrZh8o8`mF@S(|xjVKP`7m<3#~`@yw(k|W$6Gqq2xI)@ zhde3xZ+00TAm)WS;mfJ1ac8|ha9C-wjO`2~OftLu%+@ayx}*Vao59hQY8SF7@fvQr zf~q_pB8;ujtMu+weADN#(U;5af#236XZA$#CTMfmR*#%(oa$}3&H73MW|Nq>U32KLQqwCHD{|pol$X^_t7S*2JLCff6Z}g%=c2A* z-B@UdA=)_netJ4}`QdZubfm@)9i-Z0n*nfl8;sA2rK&7W>hv_eVXyPo@I$=mavvXa z$NT3c|Fm<;13@YC54)zFe*bUxTU2(ljYo5(7S9Ha3%BQ6PX8{xy5EU(j}wg8R9AtB zUYj2+3>8`(7sejr%`K$4qV)vGdp4P|ktZL2j5eBszSRX9RTnT3_bNJAhQI1BRWReU7+Y(_xCf| zYT{%8`V{|u0tV6eV)_+JQdCDZxPZ)pB2D{dtP{=tn$T4U=tkkqT-qW1pF1Bs;cpyd>2>x>#(-<4JORMDT%3gjtX>I{uwDr5`5Gw+;P)s7 zcqF-T0+Z@!z6rPDfJyW?{ZXlNxIclCuKI8<+$UjN~xf&#_j#Wim!yEZ|^<%b)-8QtWZD} z;be4{H4VU3ajAD1850nx|sZcngyaRty`JW{wsG)98+`fjCI8P9n#4y z^}m~9>sKO?i+(Tpcj8G;krpYJroJJ2MtFx}nO9b;RG1ZAyp+u*d`4Q~3!dhUFflJmClL%YW|4%X^Y3N0-7;AYWM@ygm+=yElWOS8Nd2K0?KgUl+m)L3Ouw46ns_c_Bo0@Fdcj0W*;?5E!kt+hL0$=Af5^ z-?7-KsZN3Qvs-{4)Zu%^U;kI`LZts+xwA)567}a zb1orvLQ8>N)-GN5SD&PPQ^ymdGUU(N-mUz~I+CL*a6bssWNqD7qPY^&KUG~jA$cit z5JmhJDY~b_OHI7C;bSi3=I(AOK8w8He$larAra&>qAVOz&*wW7B!BRtZb(aaJ5NLt zhyWjcV0ydz#swt*dAfmeYy!n5ul*<|eUlNu|Ijy`p`uryx(KTG-v-UsH<07&{B4W zHNs#Tpowf@Trf{prrRpjX8B>6#`oktqXl@c!Gs44aZnEX@9EsOkB?{GO{zWG9l_tw zi8z+v1G-PYBP`f**yIV^FBz)`ua3Q{XD1x78wfsobY4*8IuSOBZS@`^zn2lvflz6E z927{|I2MVfvtt@=J>;Ld=OGSb_&5)^`|NY^;*5Kz4TL&b9u`)}OW{As`Rx|P_~Pxm z{3s!Q=2A&CDv(ef4VHdhSL166_nIb5`*_=I2F?#C|oOX7xkydF9zMs6X?|0Q|u z9h2qj@yV1_QT~gidS+YL{cCt9&;T)}9sWYsKcLZ&6+U4ZCKh_BV^P!Fv>E8la#ll~ z#HQ)Cnp#38R>i&n`vsWMg1%99kF z$`+SMU2|ZqEG%5;e4zb@GI{XA^@F7jbxRQao~xT?;)1v7;LmQwXjX<d&-1HTGj~aCb=%Ll$G#-jnbQt44e8o&!?L)hA|W64v~3S_WyRFbxKa1w4hHYycIPDeSHdrHh82!p%ZP1`=U6#a3Yb-#4>T zHy^t6R!)T$ecn2n%VXapU3y5+?X&Ed`%Tac=`=PM;xt3K?!j|R;+R5HtV$bZ@xU552hDm4_o(<7#^**7k`RQW2B#DI;@zpmZ zRBexmur#B+$&p#N)w1|$MY}?mv1t~)VfZVQ&c#(}Q1tMNpyz*<@4lxyksB@0N3_R2 zzCUY2g8}*?Wb3T|M#e%wB)mdr;Ph7X*ko;in%v8bLLB)`X***Z|;eyDO5 zFVe$!o=E@QiZ@l0l$b=}_lcCy-biDI$6l>*iK(Xw9%oG=Zg~;G;Y*rkFu6|xag0K8 z|3z#B9416fR~G>t`p%gJ45Dj z!*y*CJ@*ms6XIKC!R{G%kYL!-x4|xQZ4=WJr?ZZ~cRB6#r>HSpyA1Zq1drm3a{Foq z3~TC?KsHXku3SU*T`pBO%9J~GgqRGE1&;#>yEwN%Q&7_u+sD#e4t>gnu$Ult>ak%{d3=sS10-U0WQ zV$4+*owKoaMU|nv8BxU$%nV^cfK={Ipnqd0PITcm>D;R*p0F`m62MqxaAKE{fzX>O zUR96mIt;IECBLfbACXTDf=Qg!9Sl;sIS8CTwCzq#;f9R^@Z?&c+heLKU|`F3d>TQ2 zPa;OH?^*O)ZsO2<-xm5AEczav-etT#X0m?K37f?<4%JU$oN`C?$4cF;KaUbSgmSSePyVzNcQj> zrBt%CcUxi1w(}+kE__d~R^$aVL9IMDfrLm-y^>vMd z{WB?f)f(+`U!#cv+uFa-9-eB2WL!Xt3-a`AitH#&=> zpFL7F#85CB@AcCriL3Y=cV?J4Z|`m{?w7QXHqn#$dJ`b9vD#)Hro@|ybh8Y-Qvh`} z6S2Y{+po3KZ8EnVC_2xwF@qOt0d28R)m($VxXMVmd>MI$W zaWnj(fc7bBfEModw0qm~$uE|Xo95QBu2b4ci+P1G_a(KevUT(EkH489Rvn8PVj^O4 zV#xg7z1P6jOUbBeSj*g<7tj1t%(=#*p~Dyo6P6MZqn2583S?ZETlzNSTK|n+@%vN6 zXG%8!)4HE}!~J+xD=r@*X}R@LevwyTDmwf@fx+FyNLse7Z7`n2%0(m3qqCPhEW=iS z-cr}red3=QM)Xp5a?`7WOue?3?t{zJEy8XMzJmIqTYAu$ zNxO!1&D&C}dYUslw2Lty;MKsh>lT%w&zBSv4tl|xw^YwDDMB|F5v*HUlp*xer=Q1K z0t73eDV(gtvdWEYY#aDnU2r%niu!PX*!L!XiN$GEt%_fope33EavB>wNL<6*>Hq8A ziiZ6mo7uH**Ip@W_i~#vAuXO>u5Pz+n+te|Ifp(bLaJ&XkVABqXrtZ=Kfp}Hya6@@rTl;J7#=9kKiHTx?A-uyU@ z_a_fcS7kNYt}aL^-W(@8wJ|VBcS1p8zeCRZ zlXHuLwJUqhv ztscx*=Un{#<$VlNHr5-^^{oFh;rlB$$vR-o)9fJpj-RqTh8S$TTF$U#S%DlAYe3Ob z0tpl?#?Top5GH^&wHtc^R^sH`pXBFZy5H&|COGo>%l5SIeN^wejg3MdclTd-y~@gV ztM#7v>TJ?o^UdG9o!06)^B)}h`}qm3Ipt&@G+_&(7^tDjLO1A=uU;`-R3A?O%uiEQ LSLLgcUF81*>Ykx~ literal 0 HcmV?d00001 From 3a526deae319919bd479e8bd794da02d8bf02b9c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Mon, 23 Dec 2024 23:39:23 +0100 Subject: [PATCH 185/230] shadps4 install --- functions/jsonToBashVars.sh | 3 +++ setup.sh | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/functions/jsonToBashVars.sh b/functions/jsonToBashVars.sh index dbb127d82..f89e36b56 100644 --- a/functions/jsonToBashVars.sh +++ b/functions/jsonToBashVars.sh @@ -32,6 +32,8 @@ function jsonToBashVars(){ setSetting doInstallSupermodel "$(jq .installEmus.supermodel.status $json)" setSetting doInstallModel2 "$(jq .installEmus.model2.status $json)" setSetting doInstallBigPEmu "$(jq .installEmus.bigpemu.status $json)" + setSetting doInstallShadPS4 "$(jq .installEmus.shadps4.status $json)" + #Setup Emus @@ -64,6 +66,7 @@ function jsonToBashVars(){ setSetting doSetupSupermodel "$(jq .overwriteConfigEmus.supermodel.status $json)" setSetting doSetupModel2 "$(jq .overwriteConfigEmus.model2.status $json)" setSetting doSetupBigPEmu "$(jq .overwriteConfigEmus.bigpemu.status $json)" + setSetting doSetupShadPS4 "$(jq .overwriteConfigEmus.shadps4.status $json)" #Frontends setSetting doSetupSRM "$(jq .overwriteConfigEmus.srm.status $json)" diff --git a/setup.sh b/setup.sh index ac32bb408..f27d5c8db 100644 --- a/setup.sh +++ b/setup.sh @@ -291,6 +291,11 @@ if [ "$doInstallModel2" == "true" ]; then Model2_install fi +if [ "$doInstallShadPS4" == "true" ]; then + echo "ShadPS4_install" + ShadPS4_install +fi + #Steam RomManager Config if [ "$doSetupSRM" == "true" ]; then From cb0fc2cf489b6f7b02e03e4205f52142d1fd26fc Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 20:56:09 +0100 Subject: [PATCH 186/230] shadPS4 install fix --- functions/EmuScripts/emuDeckShadPS4.sh | 109 ++++--------------------- functions/generateGameLists.sh | 22 ++--- functions/helperFunctions.sh | 27 ++++-- tools/launchers/shadps4.sh | 42 ++++++---- 4 files changed, 66 insertions(+), 134 deletions(-) diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index 18986dc49..e447e4d6f 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -16,15 +16,14 @@ # Variables ShadPS4_emuName="ShadPS4" -ShadPS4_emuType="$emuDeckEmuTypeBinary" -ShadPS4_emuPath="$HOME/Applications/publish" +ShadPS4_emuType="$emuDeckEmuTypeAppImage" +ShadPS4_emuPath="$HOME/Applications" ShadPS4_configFile="$HOME/.config/shadps4/config.toml" ShadPS4_userDir="$HOME/.config/shadps4/user" ShadPS4_sysDir="$HOME/.config/shadps4/system" ShadPS4_inputConfigDir="$HOME/.config/shadps4/inputConfig" ShadPS4_controllerFile="${ShadPS4_inputConfigDir}/default.ini" -migrationFlag="$HOME/.config/EmuDeck/.${ShadPS4_emuName}MigrationCompleted" # Language keys using [ISO 639-1: Language codes] & [ISO 3166-1 alpha-2: Country codes] # NOTE: Keep in sync with https://github.com/shadps4-emu/shadPS4/tree/main/src/qt_gui/translations @@ -111,32 +110,18 @@ ShadPS4_cleanup(){ # TODO: Install Flatpak from https://github.com/shadps4-emu/shadPS4-flatpak ShadPS4_install(){ - echo "Begin ShadPS4 Install" - local showProgress=$1 - - if installEmuBI "$ShadPS4_emuName" "$(getReleaseURLGH "ShadPS4/shadps4" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then - mkdir -p "$HOME/Applications/publish" - tar -xvf "$HOME/Applications/shadps4.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/shadps4.tar.gz" - chmod +x "$HOME/Applications/publish/shadps4" - else - return 1 - fi - - # Flatpak install - echo "Installing ShadPS4 via Flatpak..." - flatpak install flathub net.shadps4.shadPS4 -y --user - - # Move Flatpak installed files to the desired location - mkdir -p "$HOME/Applications/publish" - rsync -av "$HOME/.local/share/flatpak/app/net.shadps4.shadPS4/x86_64/stable/active/files/bin/" "$HOME/Applications/publish/" && flatpak uninstall flathub net.shadps4.shadPS4 -y --user - - # Clean up old games directory if it exists - rm -rf "$HOME/.config/shadps4/games" - - # Set executable permission - chmod +x "$HOME/Applications/publish/shadps4" + echo "Begin ShadPS4 Install" + local showProgress=$1 + + if installEmuAI "$ShadPS4_emuName" "" "$(getReleaseURLGH "shadps4-emu/shadPS4" "zip" "linux-qt")" "" "zip" "emulator" "$showProgress"; then # Cemu.AppImage + unzip -o "$HOME/Applications/ShadPS4.zip" -d "$ShadPS4_emuPath" && rm -rf "$HOME/Applications/ShadPS4.zip" + chmod +x "$ShadPS4_emuPath/publish/Shadps4.AppImage" + else + return 1 + fi } + ShadPS4_init(){ configEmuAI "$ShadPS4_emuName" "config" "$HOME/.config/shadps4" "$EMUDECKGIT/configs/shadps4" "true" ShadPS4_setupStorage @@ -193,19 +178,6 @@ ShadPS4_setEmulationFolder(){ echo "ShadPS4 Path Config Completed" } -# Reusable Function to read value from the config.toml file -read_config_toml() { - local key="$1" - local configFile="$2" - echo "Reading arguments - key '$key' from config file: '$configFile'..." - - local value - value=$(jq -r "$key" "$configFile") - - echo "Extracted value: $value" - echo "$value" -} - ShadPS4_setLanguage(){ setMSG "Setting ShadPS4 Language" local language=$(locale | grep LANG | cut -d= -f2 | cut -d_ -f1) @@ -274,46 +246,6 @@ ShadPS4_uninstall(){ uninstallGeneric $ShadPS4_emuName $ShadPS4_emuPath "" "emulator" } -# Migrate flatpak to appimage?? -ShadPS4_migrate(){ - echo "Begin ShadPS4 Migration" - - # Migration - if [ "$(ShadPS4_IsMigrated)" != "true" ]; then - #ShadPS4 flatpak to appimage - #From -- > to - migrationTable=() - migrationTable+=("$HOME/.var/app/net.shadps4.ShadPS4/config/shadps4" "$HOME/.config/shadps4") - - migrateAndLinkConfig "$ShadPS4_emuName" "$migrationTable" - fi - - echo "true" -} - -ShadPS4_IsMigrated(){ - if [ -f "$migrationFlag" ]; then - echo "true" - else - echo "false" - fi -} - -#setABXYstyle -ShadPS4_setABXYstyle(){ - sed -i 's/"button_x": "Y",/"button_x": "X",/' $ShadPS4_configFile - sed -i 's/"button_b": "A",/"button_b": "B",/' $ShadPS4_configFile - sed -i 's/"button_y": "X",/"button_y": "Y",/' $ShadPS4_configFile - sed -i 's/"button_a": "B"/"button_a": "A"/' $ShadPS4_configFile - -} -ShadPS4_setBAYXstyle(){ - sed -i 's/"button_x": "X",/"button_x": "Y",/' $ShadPS4_configFile - sed -i 's/"button_b": "B",/"button_b": "A",/' $ShadPS4_configFile - sed -i 's/"button_y": "Y",/"button_y": "X",/' $ShadPS4_configFile - sed -i 's/"button_a": "A"/"button_a": "B"/' $ShadPS4_configFile -} - #WideScreenOn ShadPS4_wideScreenOn(){ echo "NYI" @@ -340,7 +272,7 @@ ShadPS4_finalize(){ } ShadPS4_IsInstalled(){ - if [ -e "$ShadPS4_emuPath/shadps4" ]; then + if [ -e "$ShadPS4_emuPath/Shadps4-qt.AppImage" ]; then echo "true" else echo "false" @@ -352,20 +284,7 @@ ShadPS4_resetConfig(){ } ShadPS4_setResolution(){ - - case $ShadPS4Resolution in - "720P") multiplier=1; docked="false";; - "1080P") multiplier=1; docked="true";; - "1440P") multiplier=2; docked="false";; - "4K") multiplier=2; docked="true";; - *) echo "Error"; return 1;; - esac - - jq --arg docked "$docked" --arg multiplier "$multiplier" \ - '.docked_mode = $docked | .res_scale = $multiplier' "$ShadPS4_configFile" > tmp.json - - mv tmp.json "$ShadPS4_configFile" - + echo "NYI" } ShadPS4_flushEmulatorLauncher(){ diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 06f9cef3c..7104f5d64 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -2,19 +2,9 @@ MSG="$HOME/.config/EmuDeck/msg.log" -generateGameLists_pythonEnv() { - if [ ! -d "$HOME/.config/EmuDeck/python_virtual_env" ]; then - python3 -m venv "$HOME/.config/EmuDeck/python_virtual_env" - source "$HOME/.config/EmuDeck/python_virtual_env/bin/activate" - pip install requests - else - source "$HOME/.config/EmuDeck/python_virtual_env/bin/activate" - fi -} - generateGameLists() { - generateGameLists_pythonEnv &> /dev/null + generate_pythonEnv &> /dev/null local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$accountfolder/config/grid/retrolibrary/artwork/" @@ -38,7 +28,7 @@ generateGameLists() { } generateGameListsJson() { - generateGameLists_pythonEnv &> /dev/null + generate_pythonEnv &> /dev/null echo "Adding Games" > "$MSG" #python $HOME/.config/EmuDeck/backend/tools/retro-library/generate_game_lists.py "$romsPath" echo "Games Added" > "$MSG" @@ -48,12 +38,12 @@ generateGameListsJson() { } generateGameLists_importESDE() { - generateGameLists_pythonEnv &> /dev/null + generate_pythonEnv &> /dev/null python $HOME/.config/EmuDeck/backend/tools/retro-library/import_media.py "$romsPath" "$dest_folder" } generateGameLists_artwork() { - generateGameLists_pythonEnv &> /dev/null + generate_pythonEnv &> /dev/null echo "Searching for missing artwork" > "$MSG" python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_platforms.py "$romsPath" "$storagePath/retrolibrary/artwork" && python $HOME/.config/EmuDeck/backend/tools/retro-library/download_art_platforms.py "$storagePath/retrolibrary/artwork" @@ -97,7 +87,7 @@ function addGameListsArtwork() { } generateGameLists_getPercentage() { - generateGameLists_pythonEnv &> /dev/null + generate_pythonEnv &> /dev/null local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$storagePath/retrolibrary/artwork/" @@ -143,7 +133,7 @@ generateGameLists_extraArtwork() { } generateGameLists_retroAchievements(){ - generateGameLists_pythonEnv &> /dev/null + generate_pythonEnv &> /dev/null local hash=$1 local system=$2 local localDataPath="$storagePath/retrolibrary/achievements/$system.json" diff --git a/functions/helperFunctions.sh b/functions/helperFunctions.sh index 32cd4fd84..7cbed7e35 100644 --- a/functions/helperFunctions.sh +++ b/functions/helperFunctions.sh @@ -1117,10 +1117,25 @@ function startCompressor(){ konsole -e "/bin/bash $HOME/.config/EmuDeck/backend/tools/chdconv/chddeck.sh" } -function installPIP(){ - python -m pip --version &> /dev/null || python -m ensurepip --upgrade - python -m pip install --upgrade pip - grep -qxF 'PATH="$HOME/.local/bin:$PATH"' ~/.bashrc || echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc - grep -qxF 'alias pip="pip3"' ~/.bashrc || echo 'alias pip="pip3"' >> ~/.bashrc - PATH="$HOME/.local/bin:$PATH" +function generate_pythonEnv() { + if [ ! -d "$HOME/.config/EmuDeck/python_virtual_env" ]; then + python3 -m venv "$HOME/.config/EmuDeck/python_virtual_env" + source "$HOME/.config/EmuDeck/python_virtual_env/bin/activate" + pip install requests + else + source "$HOME/.config/EmuDeck/python_virtual_env/bin/activate" + fi +} + +# Reusable Function to read value from the config.toml file +function read_config_toml() { + local key="$1" + local configFile="$2" + echo "Reading arguments - key '$key' from config file: '$configFile'..." + + local value + value=$(jq -r "$key" "$configFile") + + echo "Extracted value: $value" + echo "$value" } \ No newline at end of file diff --git a/tools/launchers/shadps4.sh b/tools/launchers/shadps4.sh index ead2d7c3c..3a331f1ec 100644 --- a/tools/launchers/shadps4.sh +++ b/tools/launchers/shadps4.sh @@ -2,18 +2,18 @@ source $HOME/.config/EmuDeck/backend/functions/all.sh emulatorInit "shadps4" emuName="ShadPS4" #parameterize me -emufolder="$HOME/Applications/publish" # has to be here for ES-DE to find it +emufolder="$HOME/Applications" # has to be applications for ES-DE to find it #initialize execute array exe=() #find full path to emu executable -exe_path=$(find "$emufolder" -iname "${emuName}.sh" | sort -n | cut -d' ' -f 2- | tail -n 1 2>/dev/null) +exe_path=$(find "$emufolder" -iname "${emuName}*.AppImage" | sort -n | cut -d' ' -f 2- | tail -n 1 2>/dev/null) #if appimage doesn't exist fall back to flatpak. if [[ -z "$exe_path" ]]; then #flatpak - flatpakApp=$(flatpak list --app --columns=application | grep "$emuName") + flatpakApp=$(flatpak list --app --columns=application | grep "$RPCS3_emuPathFlatpak") #fill execute array exe=("flatpak" "run" "$flatpakApp") else @@ -23,22 +23,30 @@ else exe=("$exe_path") fi -#run the executable with the params. -launch_args=() -for rom in "${@}"; do - # Parsers previously had single quotes ("'/path/to/rom'" ), this allows those shortcuts to continue working. - removedLegacySingleQuotes=$(echo "$rom" | sed "s/^'//; s/'$//") - launch_args+=("$removedLegacySingleQuotes") -done +fileExtension="${@##*.}" -echo "Launching: ${exe[*]} ${launch_args[*]}" - -if [[ -z "${*}" ]]; then - echo "ROM not found. Launching $emuName directly" - "${exe[@]}" +if [[ $fileExtension == "desktop" ]]; then + rpcs3desktopFile=$(grep -E "^Exec=" "${*}" | sed 's/^Exec=//' | sed 's/%%/%/g') + echo "Exec=$rpcs3desktopFile" + eval $rpcs3desktopFile else - echo "ROM found, launching game" - "${exe[@]}" "${launch_args[@]}" + #run the executable with the params. + launch_args=() + for rom in "${@}"; do + # Parsers previously had single quotes ("'/path/to/rom'" ), this allows those shortcuts to continue working. + removedLegacySingleQuotes=$(echo "$rom" | sed "s/^'//; s/'$//") + launch_args+=("$removedLegacySingleQuotes") + done + + echo "Launching: ${exe[*]} ${launch_args[*]}" + + if [[ -z "${*}" ]]; then + echo "ROM not found. Launching $emuName directly" + "${exe[@]}" + else + echo "ROM found, launching game" + "${exe[@]}" "${launch_args[@]}" + fi fi rm -rf "$savesPath/.gaming" From cbb7ca237f458f8870c8b009032731f42a9e8a21 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 21:14:05 +0100 Subject: [PATCH 187/230] shadPS4 init --- .../config => }/shadps4/config.toml | 4 +- .../shadps4/inputConfig/default.ini | 0 functions/EmuScripts/emuDeckShadPS4.sh | 94 ++----------------- functions/uninstallEmuAI.sh | 4 +- tools/launchers/shadps4.sh | 2 +- 5 files changed, 15 insertions(+), 89 deletions(-) rename configs/{net.shadps4.ShadPS4/config => }/shadps4/config.toml (92%) rename configs/{net.shadps4.ShadPS4/config => }/shadps4/inputConfig/default.ini (100%) diff --git a/configs/net.shadps4.ShadPS4/config/shadps4/config.toml b/configs/shadps4/config.toml similarity index 92% rename from configs/net.shadps4.ShadPS4/config/shadps4/config.toml rename to configs/shadps4/config.toml index 8a048a0cc..c5fd284d0 100644 --- a/configs/net.shadps4.ShadPS4/config/shadps4/config.toml +++ b/configs/shadps4/config.toml @@ -57,8 +57,8 @@ geometry_x = 400 geometry_y = 400 geometry_w = 1280 geometry_h = 720 # Need to set to '800' for OLED Deck (16:10 Aspect Ratio) -installDirs = [] -addonInstallDir = "" +installDirs = "/run/media/mmcblk0p1/Emulation/storage/shadps4/games" +addonInstallDir = "/run/media/mmcblk0p1/Emulation/storage/shadps4/dlc" pkgDirs = [] elfDirs = [] recentFiles = [] diff --git a/configs/net.shadps4.ShadPS4/config/shadps4/inputConfig/default.ini b/configs/shadps4/inputConfig/default.ini similarity index 100% rename from configs/net.shadps4.ShadPS4/config/shadps4/inputConfig/default.ini rename to configs/shadps4/inputConfig/default.ini diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index e447e4d6f..bb921022e 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -1,6 +1,5 @@ #!/bin/bash -# Credits: https://github.com/Aeonitis # Script to install, initialize and configure ShadPS4 on EmuDeck # Note: No Bios/Keys symlinks necessary @@ -19,10 +18,7 @@ ShadPS4_emuName="ShadPS4" ShadPS4_emuType="$emuDeckEmuTypeAppImage" ShadPS4_emuPath="$HOME/Applications" ShadPS4_configFile="$HOME/.config/shadps4/config.toml" -ShadPS4_userDir="$HOME/.config/shadps4/user" -ShadPS4_sysDir="$HOME/.config/shadps4/system" -ShadPS4_inputConfigDir="$HOME/.config/shadps4/inputConfig" -ShadPS4_controllerFile="${ShadPS4_inputConfigDir}/default.ini" +ShadPS4_dir="$HOME/.config/shadps4/user" # Language keys using [ISO 639-1: Language codes] & [ISO 3166-1 alpha-2: Country codes] @@ -115,7 +111,7 @@ ShadPS4_install(){ if installEmuAI "$ShadPS4_emuName" "" "$(getReleaseURLGH "shadps4-emu/shadPS4" "zip" "linux-qt")" "" "zip" "emulator" "$showProgress"; then # Cemu.AppImage unzip -o "$HOME/Applications/ShadPS4.zip" -d "$ShadPS4_emuPath" && rm -rf "$HOME/Applications/ShadPS4.zip" - chmod +x "$ShadPS4_emuPath/publish/Shadps4.AppImage" + chmod +x "$ShadPS4_emuPath/publish/Shadps4-qt.AppImage" else return 1 fi @@ -123,102 +119,36 @@ ShadPS4_install(){ ShadPS4_init(){ - configEmuAI "$ShadPS4_emuName" "config" "$HOME/.config/shadps4" "$EMUDECKGIT/configs/shadps4" "true" + configEmuAI "$ShadPS4_emuName" "config" "$HOME/.local/share/shadPS4" "$EMUDECKGIT/configs/shadps4" "true" ShadPS4_setupStorage ShadPS4_setEmulationFolder ShadPS4_setupSaves ShadPS4_flushEmulatorLauncher ShadPS4_setLanguage - - # SRM_createParsers - # ShadPS4_migrate } ShadPS4_update(){ - echo "Begin ShadPS4 update" - - configEmuAI "$ShadPS4_emuName" "config" "$HOME/.config/shadps4" "$EMUDECKGIT/configs/shadps4" - - ShadPS4_setEmulationFolder - ShadPS4_setupStorage - ShadPS4_setupSaves - ShadPS4_finalize - ShadPS4_flushEmulatorLauncher + ShadPS4_init } # Configuration Paths ShadPS4_setEmulationFolder(){ echo "Begin ShadPS4 Path Config" - - # Define paths for PS4 ROMs - gameDirOpt='Paths\\gamedirs\\0\\path=' - newGameDirOpt='Paths\\gamedirs\\0\\path='"${romsPath}/ps4" - - # Update the configuration file - sed -i "/${gameDirOpt}/c\\${newGameDirOpt}" "$ShadPS4_configFile" - - # https://github.com/shadps4-emu/shadPS4/blob/3f1061de5613c0c4a74d6394a6493491280bc03f/src/common/path_util.h - mkdir -p "${ShadPS4_userDir}/screenshots/" - mkdir -p "${ShadPS4_userDir}/shader/" - mkdir -p "${ShadPS4_userDir}/savedata/" - mkdir -p "${ShadPS4_userDir}/data/" - mkdir -p "${ShadPS4_userDir}/temp/" - mkdir -p "${ShadPS4_userDir}/sys_modules/" - mkdir -p "${ShadPS4_userDir}/download/" - mkdir -p "${ShadPS4_userDir}/captures/" - mkdir -p "${ShadPS4_userDir}/cheats/" - mkdir -p "${ShadPS4_userDir}/patches/" - mkdir -p "${ShadPS4_userDir}/game_data/" - - # https://github.com/shadps4-emu/shadPS4/blob/main/documents/Debugging/Debugging.md#quick-analysis - mkdir -p "${ShadPS4_userDir}/log/" - - mkdir -p "${ShadPS4_inputConfigDir}" - + sed -i "s|/run/media/mmcblk0p1/Emulation|${emulationPath}|g" "$ShadPS4_configFile" echo "ShadPS4 Path Config Completed" } ShadPS4_setLanguage(){ setMSG "Setting ShadPS4 Language" - local language=$(locale | grep LANG | cut -d= -f2 | cut -d_ -f1) - - echo "Checking if the config file at path: '$ShadPS4_configFile'" - if [[ -f "${ShadPS4_configFile}" ]]; then - echo "Config file found: ${ShadPS4_configFile}" - - emulatorLanguage=$(read_config_toml '.GUI.emulatorLanguage' "$ShadPS4_configFile") - - echo "Checking if language key exists in current language setting..." - if [[ -n ${ShadPS4_languages[$emulatorLanguage]+_} ]]; then - echo "Language key found in current language settings!" - - # Save the updated language settings back to the config file - echo "Updating system language and system region in the config file..." - tmp=$(jq --arg lang "${ShadPS4_languages[$emulatorLanguage]}" --arg region "${ShadPS4_regions[$emulatorLanguage]}" \ - '.system_language = $lang | .system_region = $region' \ - "${ShadPS4_configFile}") - echo "$tmp" > "${ShadPS4_configFile}" - echo "Config file updated successfully." - else - echo "Language key '${emulatorLanguage}' not found in current language settings. No updates made." - fi - else - echo "Configuration file not found: ${ShadPS4_configFile}" - fi - + changeLine "emulatorLanguage = " "emulatorLanguage = ${emulatorLanguage}" $ShadPS4_configFile echo "ShadPS4 language '${emulatorLanguage}' configuration completed." } # Setup Saves ShadPS4_setupSaves(){ echo "Begin ShadPS4 save link" - # Create symbolic links - linkToSaveFolder ShadPS4 saves "${ShadPS4_userDir}/savedata" - linkToSaveFolder ShadPS4 saveMeta "${ShadPS4_userDir}/saveMeta" - linkToSaveFolder ShadPS4 system "${ShadPS4_sysDir}" - linkToSaveFolder ShadPS4 system_saves "${ShadPS4_sysDir}/save" - + linkToSaveFolder shadps4 saves "${ShadPS4_dir}/savedata" echo "ShadPS4 save link completed" } @@ -226,12 +156,8 @@ ShadPS4_setupSaves(){ #SetupStorage ShadPS4_setupStorage(){ echo "Begin ShadPS4 storage config" - - local origPath="$HOME/.config/" -# mkdir -p "${storagePath}/shadps4/patchesAndDlc" - rsync -av "${origPath}/shadps4/games/" "${storagePath}/shadps4/games/" && rm -rf "${origPath}ShadPS4/games" - unlink "${origPath}/shadps4/games" - ln -ns "${storagePath}/shadps4/games/" "${origPath}/shadps4/games" + mkdir - "$storagePath/shadps4/games" + mkdir - "$storagePath/shadps4/dlc" } #WipeSettings @@ -243,7 +169,7 @@ ShadPS4_wipe(){ #Uninstall ShadPS4_uninstall(){ echo "Begin ShadPS4 uninstall" - uninstallGeneric $ShadPS4_emuName $ShadPS4_emuPath "" "emulator" + uninstallEmuAI $ShadPS4_emuName "Shadps4-qt" "AppImage" "emulator" } #WideScreenOn diff --git a/functions/uninstallEmuAI.sh b/functions/uninstallEmuAI.sh index c9ff5f3d5..376e80e5d 100644 --- a/functions/uninstallEmuAI.sh +++ b/functions/uninstallEmuAI.sh @@ -1,6 +1,6 @@ #!/bin/bash -uninstallEmuAI() { +function uninstallEmuAI() { name=$1 filename=$2 format=$3 @@ -27,7 +27,7 @@ uninstallEmuAI() { echo "3, Application File Format: $format" echo "4, Application Type: $type" - echo "Uninstalling $name. Deleting "$HOME/Applications/$filename.$format". Deleting "$HOME/.local/share/applications/$name.desktop"" + echo "Uninstalling $name. Deleting "$HOME/Applications/$filename.$format". Deleting "$HOME/.local/share/applications/$name.desktop"" rm -rf "$HOME/Applications/$filename.$format" rm -rf "$HOME/.local/share/applications/$name.desktop" diff --git a/tools/launchers/shadps4.sh b/tools/launchers/shadps4.sh index 3a331f1ec..36dc113b9 100644 --- a/tools/launchers/shadps4.sh +++ b/tools/launchers/shadps4.sh @@ -1,7 +1,7 @@ #!/bin/bash source $HOME/.config/EmuDeck/backend/functions/all.sh emulatorInit "shadps4" -emuName="ShadPS4" #parameterize me +emuName="Shadps4-qt" #parameterize me emufolder="$HOME/Applications" # has to be applications for ES-DE to find it #initialize execute array From 4f4ddb61e995fee0ae75cce596fffde020208320 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 21:59:37 +0100 Subject: [PATCH 188/230] shadps4 --- functions/EmuScripts/emuDeckShadPS4.sh | 10 +++++----- functions/configEmuAI.sh | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index bb921022e..c3f4c25ec 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -17,8 +17,8 @@ ShadPS4_emuName="ShadPS4" ShadPS4_emuType="$emuDeckEmuTypeAppImage" ShadPS4_emuPath="$HOME/Applications" -ShadPS4_configFile="$HOME/.config/shadps4/config.toml" -ShadPS4_dir="$HOME/.config/shadps4/user" +ShadPS4_dir="$HOME/.local/share/shadPS4" +ShadPS4_configFile="$ShadPS4_dir/config.toml" # Language keys using [ISO 639-1: Language codes] & [ISO 3166-1 alpha-2: Country codes] @@ -156,14 +156,14 @@ ShadPS4_setupSaves(){ #SetupStorage ShadPS4_setupStorage(){ echo "Begin ShadPS4 storage config" - mkdir - "$storagePath/shadps4/games" - mkdir - "$storagePath/shadps4/dlc" + mkdir -p "$storagePath/shadps4/games" + mkdir -p "$storagePath/shadps4/dlc" } #WipeSettings ShadPS4_wipe(){ echo "Begin ShadPS4 delete config directories" - rm -rf "$HOME/.config/shadps4" + rm -rf "$ShadPS4_dir" } #Uninstall diff --git a/functions/configEmuAI.sh b/functions/configEmuAI.sh index 9279ca3fd..4527b1a41 100644 --- a/functions/configEmuAI.sh +++ b/functions/configEmuAI.sh @@ -1,5 +1,5 @@ #!/bin/bash -configEmuAI(){ +function configEmuAI(){ emu=$1 folderName=$2 @@ -12,8 +12,8 @@ configEmuAI(){ else overwrite="--ignore-existing" fi - - setMSG "Updating $emu $folderName using $overwrite" + + setMSG "Updating $emu $folderName using $overwrite" rsync -avhp --mkpath $gitLocation/ $folderPath $overwrite From ade9b59ba75d9e32b00d43bf4f41df95496e3518 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 22:16:47 +0100 Subject: [PATCH 189/230] shadps4 wip --- configs/shadps4/config.toml | 70 +++++++++++++------------- functions/EmuScripts/emuDeckShadPS4.sh | 4 +- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/configs/shadps4/config.toml b/configs/shadps4/config.toml index c5fd284d0..7b4964a91 100644 --- a/configs/shadps4/config.toml +++ b/configs/shadps4/config.toml @@ -1,43 +1,43 @@ -# ShadPS4 Settings/Config file - [General] -isPS4Pro = false -Fullscreen = false -playBGM = false -BGMvolume = 50 +logType = "async" +separateUpdateEnabled = false +showSplash = true +Fullscreen = true +checkCompatibilityOnStartup = true +updateChannel = "Nightly" +logFilter = "" enableDiscordRPC = false -logType = "async" # Configures logging synchronization (sync/async) -logFilter = "" # Sets the logging category for various logging classes -userName = "shadPS4" -updateChannel = "Release" -showSplash = false +compatibilityEnabled = true autoUpdate = false -separateUpdateEnabled = false +userName = "shadPS4" +isTrophyPopupDisabled = false +playBGM = false +isPS4Pro = false +BGMvolume = 50 [Input] -cursorState = 0 # HideCursorState::Idle +cursorState = 1 cursorHideTimeout = 5 backButtonBehavior = "left" -useSpecialPad = false specialPadClass = 1 +useSpecialPad = false [GPU] -# Configures the game window width and height e.g. '800' for OLED Steameck (16:10 Aspect Ratio) screenWidth = 1280 screenHeight = 720 -nullGpu = false # Disables rendering +nullGpu = false copyGPUBuffers = false -dumpShaders = false # Dump shaders that are loaded by the emulator. Dump path: ../user/shader/dumps -patchShaders = true vblankDivider = 1 +dumpShaders = false +patchShaders = true [Vulkan] gpuId = -1 validation = false +rdocMarkersEnable = false validation_sync = false validation_gpu = false -rdocEnable = false # Automatically hook RenderDoc when installed. Useful for debugging shaders and game rendering -rdocMarkersEnable = false # Enable automatic RenderDoc event annotation +rdocEnable = false crashDiagnostic = false [Debug] @@ -45,24 +45,24 @@ DebugDump = false CollectShader = false [GUI] -theme = 0 -iconSize = 36 -iconSizeGrid = 69 -sliderPos = 0 -sliderPosGrid = 0 gameTableMode = 0 mw_width = 1280 -mw_height = 720 # Need to set to '800' for OLED Deck (16:10 Aspect Ratio) -geometry_x = 400 -geometry_y = 400 -geometry_w = 1280 -geometry_h = 720 # Need to set to '800' for OLED Deck (16:10 Aspect Ratio) -installDirs = "/run/media/mmcblk0p1/Emulation/storage/shadps4/games" +geometry_x = 0 +iconSize = 36 +emulatorLanguage = "en" +iconSizeGrid = 69 +elfDirs = [] +mw_height = 718 addonInstallDir = "/run/media/mmcblk0p1/Emulation/storage/shadps4/dlc" pkgDirs = [] -elfDirs = [] -recentFiles = [] -emulatorLanguage = "en" +sliderPosGrid = 0 +geometry_h = 718 +geometry_y = 28 +recentFiles = ["/run/media/EmuDeck/Emulation/storage/shadps4/games/CUSA07010/eboot.bin"] +sliderPos = 0 +installDirs = ["/run/media/mmcblk0p1/Emulation/storage/shadps4/games"] +theme = 0 +geometry_w = 1280 [Settings] -consoleLanguage = 1 # english +consoleLanguage = 1 diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index c3f4c25ec..64a2748c0 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -140,7 +140,9 @@ ShadPS4_setEmulationFolder(){ ShadPS4_setLanguage(){ setMSG "Setting ShadPS4 Language" - changeLine "emulatorLanguage = " "emulatorLanguage = ${emulatorLanguage}" $ShadPS4_configFile + local language=$(locale | grep LANG | cut -d= -f2 | cut -d_ -f1) + #TODO: call this somewhere, and input the $language from somewhere (args?) + changeLine "emulatorLanguage = " "emulatorLanguage = ${language}" $ShadPS4_configFile echo "ShadPS4 language '${emulatorLanguage}' configuration completed." } From 3b5200f1065ea6775e8bcab480646734c9198f81 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 22:16:47 +0100 Subject: [PATCH 190/230] shadps4 wip --- configs/shadps4/config.toml | 68 +++++++++++++------------- functions/EmuScripts/emuDeckShadPS4.sh | 4 +- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/configs/shadps4/config.toml b/configs/shadps4/config.toml index c5fd284d0..68f11de13 100644 --- a/configs/shadps4/config.toml +++ b/configs/shadps4/config.toml @@ -1,43 +1,43 @@ -# ShadPS4 Settings/Config file - [General] -isPS4Pro = false -Fullscreen = false -playBGM = false -BGMvolume = 50 +logType = "async" +separateUpdateEnabled = false +showSplash = true +Fullscreen = true +checkCompatibilityOnStartup = true +updateChannel = "Nightly" +logFilter = "" enableDiscordRPC = false -logType = "async" # Configures logging synchronization (sync/async) -logFilter = "" # Sets the logging category for various logging classes -userName = "shadPS4" -updateChannel = "Release" -showSplash = false +compatibilityEnabled = true autoUpdate = false -separateUpdateEnabled = false +userName = "shadPS4" +isTrophyPopupDisabled = false +playBGM = false +isPS4Pro = false +BGMvolume = 50 [Input] -cursorState = 0 # HideCursorState::Idle +cursorState = 1 cursorHideTimeout = 5 backButtonBehavior = "left" -useSpecialPad = false specialPadClass = 1 +useSpecialPad = false [GPU] -# Configures the game window width and height e.g. '800' for OLED Steameck (16:10 Aspect Ratio) screenWidth = 1280 screenHeight = 720 -nullGpu = false # Disables rendering +nullGpu = false copyGPUBuffers = false -dumpShaders = false # Dump shaders that are loaded by the emulator. Dump path: ../user/shader/dumps -patchShaders = true vblankDivider = 1 +dumpShaders = false +patchShaders = true [Vulkan] gpuId = -1 validation = false +rdocMarkersEnable = false validation_sync = false validation_gpu = false -rdocEnable = false # Automatically hook RenderDoc when installed. Useful for debugging shaders and game rendering -rdocMarkersEnable = false # Enable automatic RenderDoc event annotation +rdocEnable = false crashDiagnostic = false [Debug] @@ -45,24 +45,24 @@ DebugDump = false CollectShader = false [GUI] -theme = 0 -iconSize = 36 -iconSizeGrid = 69 -sliderPos = 0 -sliderPosGrid = 0 gameTableMode = 0 mw_width = 1280 -mw_height = 720 # Need to set to '800' for OLED Deck (16:10 Aspect Ratio) -geometry_x = 400 -geometry_y = 400 -geometry_w = 1280 -geometry_h = 720 # Need to set to '800' for OLED Deck (16:10 Aspect Ratio) -installDirs = "/run/media/mmcblk0p1/Emulation/storage/shadps4/games" +geometry_x = 0 +iconSize = 36 +emulatorLanguage = "en" +iconSizeGrid = 69 +elfDirs = [] +mw_height = 718 addonInstallDir = "/run/media/mmcblk0p1/Emulation/storage/shadps4/dlc" pkgDirs = [] -elfDirs = [] +sliderPosGrid = 0 +geometry_h = 718 +geometry_y = 28 recentFiles = [] -emulatorLanguage = "en" +sliderPos = 0 +installDirs = ["/run/media/mmcblk0p1/Emulation/storage/shadps4/games"] +theme = 0 +geometry_w = 1280 [Settings] -consoleLanguage = 1 # english +consoleLanguage = 1 diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index c3f4c25ec..64a2748c0 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -140,7 +140,9 @@ ShadPS4_setEmulationFolder(){ ShadPS4_setLanguage(){ setMSG "Setting ShadPS4 Language" - changeLine "emulatorLanguage = " "emulatorLanguage = ${emulatorLanguage}" $ShadPS4_configFile + local language=$(locale | grep LANG | cut -d= -f2 | cut -d_ -f1) + #TODO: call this somewhere, and input the $language from somewhere (args?) + changeLine "emulatorLanguage = " "emulatorLanguage = ${language}" $ShadPS4_configFile echo "ShadPS4 language '${emulatorLanguage}' configuration completed." } From 7c4ea7561b99c01f796843416fdaf75b664a374f Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 22:19:32 +0100 Subject: [PATCH 191/230] lang --- functions/EmuScripts/emuDeckShadPS4.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index 64a2748c0..d2b3b63cc 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -142,7 +142,7 @@ ShadPS4_setLanguage(){ setMSG "Setting ShadPS4 Language" local language=$(locale | grep LANG | cut -d= -f2 | cut -d_ -f1) #TODO: call this somewhere, and input the $language from somewhere (args?) - changeLine "emulatorLanguage = " "emulatorLanguage = ${language}" $ShadPS4_configFile + changeLine "emulatorLanguage = " "emulatorLanguage = \"${language}\"" $ShadPS4_configFile echo "ShadPS4 language '${emulatorLanguage}' configuration completed." } From bd9767c95172c3b721de2f1cb984f16ebe605c71 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 22:39:28 +0100 Subject: [PATCH 192/230] no check 4 compat --- configs/shadps4/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/shadps4/config.toml b/configs/shadps4/config.toml index 68f11de13..0ddd9426b 100644 --- a/configs/shadps4/config.toml +++ b/configs/shadps4/config.toml @@ -3,7 +3,7 @@ logType = "async" separateUpdateEnabled = false showSplash = true Fullscreen = true -checkCompatibilityOnStartup = true +checkCompatibilityOnStartup = false updateChannel = "Nightly" logFilter = "" enableDiscordRPC = false From 246e853ae66d503a946c8b144e9f9bf8356b48bc Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 22:40:05 +0100 Subject: [PATCH 193/230] PS4 parser --- .../userData/userConfigurations.json | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index 9c80c1ea0..97e551a16 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9215,6 +9215,90 @@ "Sony PlayStation 3" ] }, + { + "parserType": "Glob-regex", + "configTitle": "Sony PlayStation 4 - ShadPS4 (Installed PKG)", + "steamDirectory": "${steamdirglobal}", + "romDirectory": "/run/media/mmcblk0p1/Emulation/storage/shadps4/games", + "executableArgs": "--g \"'${filePath}'\"", + "executableModifier": "\"${exePath}\"", + "startInDirectory": "", + "titleModifier": "${fuzzyTitle}", + "imageProviders": [ + "sgdb", + "steamCDN" + ], + "onlineImageQueries": "${${fuzzyTitle}}", + "imagePool": "${fuzzyTitle}", + "disabled": false, + "userAccounts": { + "specifiedAccounts": [ + "Global" + ] + }, + "controllers": { + "ps4": null, + "ps5": null, + "xbox360": null, + "xboxone": null, + "switch_joycon_left": null, + "switch_joycon_right": null, + "switch_pro": null, + "neptune": null + }, + "executable": { + "path": "/run/media/mmcblk0p1/Emulation/tools/launchers/shadps4.sh", + "shortcutPassthrough": false, + "appendArgsToExecutable": true + }, + "parserInputs": { + "glob-regex": "${/(^[NP].+)/}/@(eboot.bin|EBOOT.BIN)" + }, + "titleFromVariable": { + "limitToGroups": "${PS4}", + "caseInsensitiveVariables": false, + "skipFileIfVariableWasNotFound": false + }, + "fuzzyMatch": { + "replaceDiacritics": true, + "removeCharacters": true, + "removeBrackets": true + }, + "parserId": "165182011309044893", + "version": 22, + "imageProviderAPIs": { + "sgdb": { + "nsfw": false, + "humor": false, + "imageMotionTypes": [ + "static" + ], + "styles": [], + "stylesHero": [], + "stylesLogo": [], + "stylesIcon": [] + } + }, + "defaultImage": { + "long": "", + "tall": "", + "hero": "", + "logo": "", + "icon": "/home/deck/.config/EmuDeck/backend/configs/steam-rom-manager/userData/img/default/icon.png" + }, + "localImages": { + "long": "", + "tall": "", + "hero": "", + "logo": "", + "icon": "" + }, + "steamInputEnabled": "1", + "drmProtect": false, + "steamCategories": [ + "Sony PlayStation 4" + ] + }, { "parserType": "Glob", "configTitle": "Sony PlayStation Portable - PPSSPP (Standalone)", From 8947372f434e6abcf7f22237da72730140aca5f9 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 23:21:47 +0100 Subject: [PATCH 194/230] fixes launch in game mode --- configs/shadps4/config.toml | 2 +- configs/steam-rom-manager/userData/userConfigurations.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/shadps4/config.toml b/configs/shadps4/config.toml index 0ddd9426b..40ae4b80b 100644 --- a/configs/shadps4/config.toml +++ b/configs/shadps4/config.toml @@ -1,7 +1,7 @@ [General] logType = "async" separateUpdateEnabled = false -showSplash = true +showSplash = false Fullscreen = true checkCompatibilityOnStartup = false updateChannel = "Nightly" diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index 97e551a16..b8c820f42 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9220,7 +9220,7 @@ "configTitle": "Sony PlayStation 4 - ShadPS4 (Installed PKG)", "steamDirectory": "${steamdirglobal}", "romDirectory": "/run/media/mmcblk0p1/Emulation/storage/shadps4/games", - "executableArgs": "--g \"'${filePath}'\"", + "executableArgs": "-g \"'${filePath}'\"", "executableModifier": "\"${exePath}\"", "startInDirectory": "", "titleModifier": "${fuzzyTitle}", @@ -9230,7 +9230,7 @@ ], "onlineImageQueries": "${${fuzzyTitle}}", "imagePool": "${fuzzyTitle}", - "disabled": false, + "disabled": true, "userAccounts": { "specifiedAccounts": [ "Global" @@ -9252,7 +9252,7 @@ "appendArgsToExecutable": true }, "parserInputs": { - "glob-regex": "${/(^[NP].+)/}/@(eboot.bin|EBOOT.BIN)" + "glob-regex": "${/(^.+)/}/@(eboot.bin|EBOOT.BIN)" }, "titleFromVariable": { "limitToGroups": "${PS4}", From af930350e22a62147e631d9204db61a22b0866eb Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 23:22:55 +0100 Subject: [PATCH 195/230] 0.4 frozen --- functions/EmuScripts/emuDeckShadPS4.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index d2b3b63cc..3a89647e6 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -109,7 +109,9 @@ ShadPS4_install(){ echo "Begin ShadPS4 Install" local showProgress=$1 - if installEmuAI "$ShadPS4_emuName" "" "$(getReleaseURLGH "shadps4-emu/shadPS4" "zip" "linux-qt")" "" "zip" "emulator" "$showProgress"; then # Cemu.AppImage + #if installEmuAI "$ShadPS4_emuName" "" "$(getReleaseURLGH "shadps4-emu/shadPS4" "zip" "linux-qt")" "" "zip" "emulator" "$showProgress"; then + + if installEmuAI "$ShadPS4_emuName" "" "https://github.com/shadps4-emu/shadPS4/releases/download/v.0.4.0/shadps4-linux-qt-0.4.0.zip" "" "zip" "emulator" "$showProgress"; then unzip -o "$HOME/Applications/ShadPS4.zip" -d "$ShadPS4_emuPath" && rm -rf "$HOME/Applications/ShadPS4.zip" chmod +x "$ShadPS4_emuPath/publish/Shadps4-qt.AppImage" else From 9cd9cefce0dc3742cf0010996a11db4996ac0097 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 23:35:34 +0100 Subject: [PATCH 196/230] shadPS4 cleanup --- functions/EmuScripts/emuDeckShadPS4.sh | 92 +------------------------- 1 file changed, 1 insertion(+), 91 deletions(-) diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index 3a89647e6..8c1597076 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -2,104 +2,14 @@ # Script to install, initialize and configure ShadPS4 on EmuDeck # Note: No Bios/Keys symlinks necessary - -# External helper functions (defined outside this script) -#- installEmuBI() -#- getReleaseURLGH() -#- configEmuAI() -#- linkToSaveFolder() -#- uninstallGeneric() -#- migrateAndLinkConfig() -#- flushEmulatorLaunchers() -#- setMSG() - # Variables + ShadPS4_emuName="ShadPS4" ShadPS4_emuType="$emuDeckEmuTypeAppImage" ShadPS4_emuPath="$HOME/Applications" ShadPS4_dir="$HOME/.local/share/shadPS4" ShadPS4_configFile="$ShadPS4_dir/config.toml" - -# Language keys using [ISO 639-1: Language codes] & [ISO 3166-1 alpha-2: Country codes] -# NOTE: Keep in sync with https://github.com/shadps4-emu/shadPS4/tree/main/src/qt_gui/translations -# even though project still just uses some two character codes e.g. 'nl' instead of 'nl_NL' -declare -A ShadPS4_languages -ShadPS4_languages=( - ["ar"]="Arabic" - ["da_DK"]="Danish" - ["de"]="German Deutsch" - ["el"]="Greek" - ["el_GR"]="Greek" - ["en"]="English" - ["en_US"]="English (US)" - ["en_IE"]="English (Irish)" - ["es_ES"]="Spanish" - ["fa_IR"]="Farsi (Iran)" - ["fi"]="Finnish" - ["fi_FI"]="Finnish" - ["fr"]="French" - ["fr_FR"]="French" - ["hu_HU"]="Hungarian" - ["id"]="Indonesian" - ["it"]="Italian" - ["ja_JP"]="Japanese" - ["ko_KR"]="Korean" - ["lt_LT"]="Lithuanian" - ["nb"]="Norwegian Bokmål" - ["nl"]="Dutch" - ["nl_NL"]="Dutch (Netherlands)" - ["pl_PL"]="Polish" - ["pt_BR"]="Portuguese" - ["ro_RO"]="Romanian" - ["ru_RU"]="Russian" - ["sq"]="Albanian" - ["ti_ER"]="Tigrinya" - ["tr_TR"]="Turkish" - ["uk_UA"]="Ukrainian" - ["vi_VN"]="Vietnamese" - ["zh_CN"]="Chinese (Simplified)" - ["zh_TW"]="Traditional Chinese (Taiwan)" -) - -declare -A ShadPS4_regions -ShadPS4_regions=( - ["ar"]="Arabic" - ["da_DK"]="Denmark" - ["de"]="Deutsch" - ["el"]="Greece" - ["el_GR"]="Greece" - ["en"]="Global English" - ["en_US"]="United States" - ["en_IE"]="Ireland" - ["es_ES"]="Spain" - ["fa_IR"]="Iran" - ["fi"]="Finland" - ["fi_FI"]="Finland" - ["fr"]="France" - ["fr_FR"]="France" - ["hu_HU"]="Hungary" - ["id"]="Indonesia" - ["it"]="Italian" - ["ja_JP"]="Japan" - ["ko_KR"]="South Korea" - ["lt_LT"]="Lithuania" - ["nb"]="Norway" - ["nl"]="Netherlands" - ["nl_NL"]="Netherlands" - ["pl_PL"]="Poland" - ["pt_BR"]="Brazil" - ["ro_RO"]="Romania" - ["ru_RU"]="Russia" - ["sq"]="Albania" - ["ti_ER"]="Eritrea" - ["tr_TR"]="Turkey" - ["uk_UA"]="Ukraine" - ["vi_VN"]="Vietnam" - ["zh_CN"]="China" - ["zh_TW"]="Taiwan" -) - ShadPS4_cleanup(){ echo "Begin ShadPS4 Cleanup" } From 465a4b6744238a8771d4350c1f030e917790310e Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Wed, 25 Dec 2024 23:36:10 +0100 Subject: [PATCH 197/230] old save function back --- functions/helperFunctions.sh | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/functions/helperFunctions.sh b/functions/helperFunctions.sh index 7cbed7e35..e6b854e45 100644 --- a/functions/helperFunctions.sh +++ b/functions/helperFunctions.sh @@ -429,26 +429,30 @@ function getReleaseURLGH(){ } function linkToSaveFolder(){ - local emu=$1 # /saves/$emu/ - local folderName=$2 # states or saves - local path=$3 # original location - - mkdir -p "$path" + local emu=$1 + local folderName=$2 + local path=$3 - # Es el path un simlink? - if [ ! -L "$path" ]; then - #We delete the old symlink and move the saves - if [ -L "$savesPath/$emu/$folderName" ]; then + if [ ! -d "$savesPath/$emu/$folderName" ]; then + if [ ! -L "$savesPath/$emu/$folderName" ]; then + mkdir -p "$savesPath/$emu" setMSG "Linking $emu $folderName to the Emulation/saves folder" - rm -rf "$savesPath/$emu/$folderName" - mv "$path" "$savesPath/$emu/$folderName" - ln -snv "$savesPath/$emu/$folderName" "$path" + mkdir -p "$path" + ln -snv "$path" "$savesPath/$emu/$folderName" fi else - setMSG "Rebuilding Link $emu $folderName to the Emulation/saves folder" - mkdir -p "$savesPath/$emu/$folderName" - unlink "$savesPath/$emu/$folderName" - ln -snv "$savesPath/$emu/$folderName" "$path" + if [ ! -L "$savesPath/$emu/$folderName" ]; then + echo "$savesPath/$emu/$folderName is not a link. Please check it." + else + if [ $(readlink $savesPath/$emu/$folderName) == $path ]; then + echo "$savesPath/$emu/$folderName is already linked." + echo " Target: $(readlink $savesPath/$emu/$folderName)" + else + echo "$savesPath/$emu/$folderName not linked correctly." + unlink "$savesPath/$emu/$folderName" + linkToSaveFolder "$emu" "$folderName" "$path" + fi + fi fi } From 71369cbd926692977b104dac53cf40e7b9821f31 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 01:05:17 +0100 Subject: [PATCH 198/230] shadps4 parser --- configs/steam-rom-manager/userData/userConfigurations.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index b8c820f42..5a38bcc24 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9252,7 +9252,7 @@ "appendArgsToExecutable": true }, "parserInputs": { - "glob-regex": "${/(^.+)/}/@(eboot.bin|EBOOT.BIN)" + "glob-regex": "${/(.+)/}/@(eboot.bin|EBOOT.BIN)" }, "titleFromVariable": { "limitToGroups": "${PS4}", From 9abfc60d091fb15f7bc997bd9db6ce391f5ba74b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 02:18:02 +0100 Subject: [PATCH 199/230] ps4 ESDE --- .../custom_systems/es_find_rules.xml | 6 ++ .../custom_systems/es_systems.xml | 18 ++++ .../userData/userConfigurations.json | 86 ++++++++++++++++++- roms/ps4/shortcuts/readme.txt | 1 + 4 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 roms/ps4/shortcuts/readme.txt diff --git a/configs/emulationstation/custom_systems/es_find_rules.xml b/configs/emulationstation/custom_systems/es_find_rules.xml index 30ce063b9..73c56fced 100644 --- a/configs/emulationstation/custom_systems/es_find_rules.xml +++ b/configs/emulationstation/custom_systems/es_find_rules.xml @@ -33,6 +33,12 @@ /usr/pkg/lib/libretro + + + + ~/Applications/Shadps4*.AppImage + + diff --git a/configs/emulationstation/custom_systems/es_systems.xml b/configs/emulationstation/custom_systems/es_systems.xml index 272247ebc..75ca2c3fc 100644 --- a/configs/emulationstation/custom_systems/es_systems.xml +++ b/configs/emulationstation/custom_systems/es_systems.xml @@ -58,4 +58,22 @@ xbox360 xbox360 + + ps4 + Sony PlayStation 4 + %ROMPATH%/ps4/shortcuts + .desktop + %HIDEWINDOW% %ESCAPESPECIALS% %EMULATOR_OS-SHELL% %ROM% + ps4 + ps4 + + diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index 5a38bcc24..9027b1ad6 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9220,7 +9220,7 @@ "configTitle": "Sony PlayStation 4 - ShadPS4 (Installed PKG)", "steamDirectory": "${steamdirglobal}", "romDirectory": "/run/media/mmcblk0p1/Emulation/storage/shadps4/games", - "executableArgs": "-g \"'${filePath}'\"", + "executableArgs": "\"'${filePath}'\"", "executableModifier": "\"${exePath}\"", "startInDirectory": "", "titleModifier": "${fuzzyTitle}", @@ -9299,6 +9299,90 @@ "Sony PlayStation 4" ] }, + { + "parserType": "Glob", + "configTitle": "Sony PlayStation 4 - ShadPS4 (Shortcut)", + "steamDirectory": "${steamdirglobal}", + "romDirectory": "${romsdirglobal}/ps4/shortcuts", + "executableArgs": "-batch -fullscreen \"'${filePath}'\"", + "executableModifier": "\"${exePath}\"", + "startInDirectory": "", + "titleModifier": "${fuzzyTitle}", + "imageProviders": [ + "sgdb", + "steamCDN" + ], + "onlineImageQueries": "${${fuzzyTitle}}", + "imagePool": "${fuzzyTitle}", + "disabled": false, + "userAccounts": { + "specifiedAccounts": [ + "Global" + ] + }, + "executable": { + "path": "/run/media/mmcblk0p1/Emulation/tools/launchers/shadps4.sh", + "shortcutPassthrough": false, + "appendArgsToExecutable": false + }, + "parserInputs": { + "glob": "**/@(.desktop)" + }, + "titleFromVariable": { + "limitToGroups": "", + "caseInsensitiveVariables": false, + "skipFileIfVariableWasNotFound": false + }, + "fuzzyMatch": { + "replaceDiacritics": true, + "removeCharacters": true, + "removeBrackets": true + }, + "imageProviderAPIs": { + "sgdb": { + "nsfw": false, + "humor": false, + "styles": [], + "stylesHero": [], + "stylesLogo": [], + "stylesIcon": [], + "imageMotionTypes": [ + "static" + ] + } + }, + "parserId": "164712748872922785", + "version": 22, + "controllers": { + "ps4": null, + "ps5": null, + "xbox360": null, + "xboxone": null, + "switch_joycon_left": null, + "switch_joycon_right": null, + "switch_pro": null, + "neptune": null + }, + "defaultImage": { + "long": "", + "tall": "", + "hero": "", + "logo": "", + "icon": "/home/deck/.config/EmuDeck/backend/configs/steam-rom-manager/userData/img/default/icon.png" + }, + "localImages": { + "long": "", + "tall": "", + "hero": "", + "logo": "", + "icon": "" + }, + "steamInputEnabled": "1", + "drmProtect": false, + "steamCategories": [ + "Sony PlayStation 4" + ] + }, { "parserType": "Glob", "configTitle": "Sony PlayStation Portable - PPSSPP (Standalone)", diff --git a/roms/ps4/shortcuts/readme.txt b/roms/ps4/shortcuts/readme.txt new file mode 100644 index 000000000..9636535f6 --- /dev/null +++ b/roms/ps4/shortcuts/readme.txt @@ -0,0 +1 @@ +Place your .desktop files here. \ No newline at end of file From ce82531549679f4e6b4661dea4fc205fc3a3650c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 02:34:52 +0100 Subject: [PATCH 200/230] fixes --- configs/steam-rom-manager/userData/userConfigurations.json | 2 +- tools/launchers/shadps4.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index 9027b1ad6..1dbe1778a 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9326,7 +9326,7 @@ "appendArgsToExecutable": false }, "parserInputs": { - "glob": "**/@(.desktop)" + "glob": "**/${title}@(.desktop)" }, "titleFromVariable": { "limitToGroups": "", diff --git a/tools/launchers/shadps4.sh b/tools/launchers/shadps4.sh index 36dc113b9..81f2be1cc 100644 --- a/tools/launchers/shadps4.sh +++ b/tools/launchers/shadps4.sh @@ -27,8 +27,9 @@ fileExtension="${@##*.}" if [[ $fileExtension == "desktop" ]]; then rpcs3desktopFile=$(grep -E "^Exec=" "${*}" | sed 's/^Exec=//' | sed 's/%%/%/g') - echo "Exec=$rpcs3desktopFile" - eval $rpcs3desktopFile + launchParam="Exec=$rpcs3desktopFile" + launchParam=$(echo "$launchParam" | sed "s|^\(Exec=\)[^\"']*\"|\1$HOME/Applications/Shadps4-qt.AppImage\"|") + eval $launchParam else #run the executable with the params. launch_args=() From a421fc86edbcd3795440c8f8cf9d9470c4150a0d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 02:49:19 +0100 Subject: [PATCH 201/230] fix batch --- configs/steam-rom-manager/userData/userConfigurations.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index 1dbe1778a..5f1188486 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9304,7 +9304,7 @@ "configTitle": "Sony PlayStation 4 - ShadPS4 (Shortcut)", "steamDirectory": "${steamdirglobal}", "romDirectory": "${romsdirglobal}/ps4/shortcuts", - "executableArgs": "-batch -fullscreen \"'${filePath}'\"", + "executableArgs": "\"'${filePath}'\"", "executableModifier": "\"${exePath}\"", "startInDirectory": "", "titleModifier": "${fuzzyTitle}", From cacf06fb6cba742a7cad3f53368c759c64b615ac Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 02:58:05 +0100 Subject: [PATCH 202/230] wip --- configs/emulationstation/custom_systems/es_find_rules.xml | 2 +- configs/emulationstation/custom_systems/es_systems.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/emulationstation/custom_systems/es_find_rules.xml b/configs/emulationstation/custom_systems/es_find_rules.xml index 73c56fced..c96d8dbbc 100644 --- a/configs/emulationstation/custom_systems/es_find_rules.xml +++ b/configs/emulationstation/custom_systems/es_find_rules.xml @@ -36,7 +36,7 @@ - ~/Applications/Shadps4*.AppImage + /run/media/mmcblk0p1/Emulation/tools/launchers/shadps4.sh diff --git a/configs/emulationstation/custom_systems/es_systems.xml b/configs/emulationstation/custom_systems/es_systems.xml index 75ca2c3fc..33b25807f 100644 --- a/configs/emulationstation/custom_systems/es_systems.xml +++ b/configs/emulationstation/custom_systems/es_systems.xml @@ -63,7 +63,7 @@ Sony PlayStation 4 %ROMPATH%/ps4/shortcuts .desktop - %HIDEWINDOW% %ESCAPESPECIALS% %EMULATOR_OS-SHELL% %ROM% + %EMULATOR_SHADPS4% %ROM% ps4 ps4 From e26508aea433750b0932946f06c1daa16225965c Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 03:07:17 +0100 Subject: [PATCH 203/230] custom rules ESDE --- .../emulationstation/custom_systems/es_find_rules.xml | 6 ++++++ functions/ToolScripts/emuDeckESDE.sh | 1 + 2 files changed, 7 insertions(+) diff --git a/chimeraOS/configs/emulationstation/custom_systems/es_find_rules.xml b/chimeraOS/configs/emulationstation/custom_systems/es_find_rules.xml index bea486e59..89ee19258 100644 --- a/chimeraOS/configs/emulationstation/custom_systems/es_find_rules.xml +++ b/chimeraOS/configs/emulationstation/custom_systems/es_find_rules.xml @@ -41,6 +41,12 @@ /usr/pkg/lib/libretro + + + + /run/media/mmcblk0p1/Emulation/tools/launchers/shadps4.sh + + diff --git a/functions/ToolScripts/emuDeckESDE.sh b/functions/ToolScripts/emuDeckESDE.sh index f1090bac6..6429e307e 100644 --- a/functions/ToolScripts/emuDeckESDE.sh +++ b/functions/ToolScripts/emuDeckESDE.sh @@ -116,6 +116,7 @@ ESDE_init(){ rsync -avhp --mkpath "$EMUDECKGIT/chimeraOS/configs/emulationstation/custom_systems/es_find_rules.xml" "$(dirname "$es_rulesFile")" --backup --suffix=.bak # This duplicates ESDE_addCustomSystemsFile but this line only applies only if you are resetting ES-DE and not the emulators themselves. rsync -avhp --mkpath "$EMUDECKGIT/configs/emulationstation/custom_systems/es_systems.xml" "$(dirname "$es_systemsFile")" --backup --suffix=.bak + sed -i "s|/run/media/mmcblk0p1/Emulation|${emulationPath}|g" "$es_rulesFile" ESDE_createLauncher ESDE_addCustomSystems From a879cd4a0c19e8fd07a5474a11b5342a65804e88 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 10:56:24 +0100 Subject: [PATCH 204/230] shadps4 last release --- functions/EmuScripts/emuDeckShadPS4.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/functions/EmuScripts/emuDeckShadPS4.sh b/functions/EmuScripts/emuDeckShadPS4.sh index 8c1597076..b6c886957 100644 --- a/functions/EmuScripts/emuDeckShadPS4.sh +++ b/functions/EmuScripts/emuDeckShadPS4.sh @@ -19,9 +19,7 @@ ShadPS4_install(){ echo "Begin ShadPS4 Install" local showProgress=$1 - #if installEmuAI "$ShadPS4_emuName" "" "$(getReleaseURLGH "shadps4-emu/shadPS4" "zip" "linux-qt")" "" "zip" "emulator" "$showProgress"; then - - if installEmuAI "$ShadPS4_emuName" "" "https://github.com/shadps4-emu/shadPS4/releases/download/v.0.4.0/shadps4-linux-qt-0.4.0.zip" "" "zip" "emulator" "$showProgress"; then + if installEmuAI "$ShadPS4_emuName" "" "$(getReleaseURLGH "shadps4-emu/shadPS4" "zip" "linux-qt")" "" "zip" "emulator" "$showProgress"; then unzip -o "$HOME/Applications/ShadPS4.zip" -d "$ShadPS4_emuPath" && rm -rf "$HOME/Applications/ShadPS4.zip" chmod +x "$ShadPS4_emuPath/publish/Shadps4-qt.AppImage" else From 190fd4f7c1a213c72d7f1470de349985049c0816 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 11:03:45 +0100 Subject: [PATCH 205/230] no Exec --- tools/launchers/shadps4.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/launchers/shadps4.sh b/tools/launchers/shadps4.sh index 81f2be1cc..240c26e6b 100644 --- a/tools/launchers/shadps4.sh +++ b/tools/launchers/shadps4.sh @@ -28,7 +28,7 @@ fileExtension="${@##*.}" if [[ $fileExtension == "desktop" ]]; then rpcs3desktopFile=$(grep -E "^Exec=" "${*}" | sed 's/^Exec=//' | sed 's/%%/%/g') launchParam="Exec=$rpcs3desktopFile" - launchParam=$(echo "$launchParam" | sed "s|^\(Exec=\)[^\"']*\"|\1$HOME/Applications/Shadps4-qt.AppImage\"|") + launchParam=$(echo "$launchParam" | sed "s|^\(Exec=\)[^\"']*\"|\1$HOME/Applications/Shadps4-qt.AppImage -g \"|" | sed 's/^Exec=//') eval $launchParam else #run the executable with the params. From 766639ec9024798ea81936de416cd16f474737e3 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 11:05:50 +0100 Subject: [PATCH 206/230] no ' --- configs/steam-rom-manager/userData/userConfigurations.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index 5f1188486..8462b725f 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9304,7 +9304,7 @@ "configTitle": "Sony PlayStation 4 - ShadPS4 (Shortcut)", "steamDirectory": "${steamdirglobal}", "romDirectory": "${romsdirglobal}/ps4/shortcuts", - "executableArgs": "\"'${filePath}'\"", + "executableArgs": "\"${filePath}\"", "executableModifier": "\"${exePath}\"", "startInDirectory": "", "titleModifier": "${fuzzyTitle}", From 8c2909f65e148a704a4958c90237f637a0d62dc9 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 11:14:06 +0100 Subject: [PATCH 207/230] Metadata --- roms/ps4/metadata.txt | 6 ++---- roms/ps4/systeminfo.txt | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/roms/ps4/metadata.txt b/roms/ps4/metadata.txt index 7f17eafd3..1d138dfb3 100644 --- a/roms/ps4/metadata.txt +++ b/roms/ps4/metadata.txt @@ -1,6 +1,4 @@ collection: Sony PlayStation 4 shortname: ps4 -extensions: 7z, 7Z, zip, ZIP -launch: PLACEHOLDER {file.path} - - +extensions: desktop +launch: /run/media/mmcblk0p1/Emulation/tools/launchers/shadps4.sh {file.path} \ No newline at end of file diff --git a/roms/ps4/systeminfo.txt b/roms/ps4/systeminfo.txt index b071bcd67..a85ea6ff8 100644 --- a/roms/ps4/systeminfo.txt +++ b/roms/ps4/systeminfo.txt @@ -5,7 +5,7 @@ Full system name: Sony PlayStation 4 Supported file extensions: -.7z .7Z .zip .ZIP +.desktop Launch command: PLACEHOLDER %ROM% From 9c45f61a2286ad9c89c5d8ed05ec859efd42b844 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 11:14:29 +0100 Subject: [PATCH 208/230] no pkg --- .../userData/userConfigurations.json | 84 ------------------- 1 file changed, 84 deletions(-) diff --git a/configs/steam-rom-manager/userData/userConfigurations.json b/configs/steam-rom-manager/userData/userConfigurations.json index 8462b725f..ac3940eef 100644 --- a/configs/steam-rom-manager/userData/userConfigurations.json +++ b/configs/steam-rom-manager/userData/userConfigurations.json @@ -9215,90 +9215,6 @@ "Sony PlayStation 3" ] }, - { - "parserType": "Glob-regex", - "configTitle": "Sony PlayStation 4 - ShadPS4 (Installed PKG)", - "steamDirectory": "${steamdirglobal}", - "romDirectory": "/run/media/mmcblk0p1/Emulation/storage/shadps4/games", - "executableArgs": "\"'${filePath}'\"", - "executableModifier": "\"${exePath}\"", - "startInDirectory": "", - "titleModifier": "${fuzzyTitle}", - "imageProviders": [ - "sgdb", - "steamCDN" - ], - "onlineImageQueries": "${${fuzzyTitle}}", - "imagePool": "${fuzzyTitle}", - "disabled": true, - "userAccounts": { - "specifiedAccounts": [ - "Global" - ] - }, - "controllers": { - "ps4": null, - "ps5": null, - "xbox360": null, - "xboxone": null, - "switch_joycon_left": null, - "switch_joycon_right": null, - "switch_pro": null, - "neptune": null - }, - "executable": { - "path": "/run/media/mmcblk0p1/Emulation/tools/launchers/shadps4.sh", - "shortcutPassthrough": false, - "appendArgsToExecutable": true - }, - "parserInputs": { - "glob-regex": "${/(.+)/}/@(eboot.bin|EBOOT.BIN)" - }, - "titleFromVariable": { - "limitToGroups": "${PS4}", - "caseInsensitiveVariables": false, - "skipFileIfVariableWasNotFound": false - }, - "fuzzyMatch": { - "replaceDiacritics": true, - "removeCharacters": true, - "removeBrackets": true - }, - "parserId": "165182011309044893", - "version": 22, - "imageProviderAPIs": { - "sgdb": { - "nsfw": false, - "humor": false, - "imageMotionTypes": [ - "static" - ], - "styles": [], - "stylesHero": [], - "stylesLogo": [], - "stylesIcon": [] - } - }, - "defaultImage": { - "long": "", - "tall": "", - "hero": "", - "logo": "", - "icon": "/home/deck/.config/EmuDeck/backend/configs/steam-rom-manager/userData/img/default/icon.png" - }, - "localImages": { - "long": "", - "tall": "", - "hero": "", - "logo": "", - "icon": "" - }, - "steamInputEnabled": "1", - "drmProtect": false, - "steamCategories": [ - "Sony PlayStation 4" - ] - }, { "parserType": "Glob", "configTitle": "Sony PlayStation 4 - ShadPS4 (Shortcut)", From 6798443a228c263c385286f312cbaca5aebeb399 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 11:25:50 +0100 Subject: [PATCH 209/230] update & uninstall shadps4 --- tools/binupdate/binupdate.sh | 16 +++++ uninstall.sh | 114 ++++++++++++++++++++--------------- 2 files changed, 80 insertions(+), 50 deletions(-) diff --git a/tools/binupdate/binupdate.sh b/tools/binupdate/binupdate.sh index e7ea608e7..903fc4b92 100755 --- a/tools/binupdate/binupdate.sh +++ b/tools/binupdate/binupdate.sh @@ -168,6 +168,17 @@ function runBinDownloads { messages+=("There was a problem updating RPCS3") fi fi + if [[ "$binsToDL" == *"ShadPS4"* ]]; then + ((progresspct += pct)) || true + echo "$progresspct" + echo "# Updating ShadPS4" + if ShadPS4_install "true" 2>&1; then + messages+=("ShadPS4 Updated Successfully") + else + messages+=("There was a problem updating ShadPS4") + fi + fi + if [[ "$binsToDL" == *"Ryujinx"* ]]; then ((progresspct += pct)) || true echo "$progresspct" @@ -269,6 +280,11 @@ if [ "$(RPCS3_IsInstalled ""$emuDeckEmuTypeAppImage"")" == "true" ]; then else binTable+=(FALSE "RPCS3" "Sony PlayStation 3") fi +if [ "$(ShadPS4_IsInstalled ""$emuDeckEmuTypeAppImage"")" == "true" ]; then + binTable+=(TRUE "ShadPS4" "Sony PlayStation 4") +else + binTable+=(FALSE "ShadPS4" "Sony PlayStation 4") +fi if [ "$(Ryujinx_IsInstalled ""$emuDeckEmuTypeBinary"")" == "true" ]; then binTable+=(TRUE "Ryujinx" "Nintendo Switch") else diff --git a/uninstall.sh b/uninstall.sh index 99423ca25..e74ba3ef2 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -22,6 +22,7 @@ doUninstallPPSSPP=true doUninstallPrimeHacks=true doUninstallRMG=true doUninstallRPCS3=true +doUninstallShadPS4=true doUninstallRyujinx=true doUninstallScummVM=true doUninstallSRM=true @@ -154,50 +155,50 @@ fi flatpak uninstall re.chiaki.Chiaki -y rm -f "$romsPath/remoteplay/Chiaki Remote Play Client.sh" &> /dev/null rm -rf "$HOME/.var/app/re.chiaki.Chiaki" &> /dev/null - fi + fi - if [[ "$RPUninstall" != *"chiaki-ng"* ]]; then + if [[ "$RPUninstall" != *"chiaki-ng"* ]]; then rm -f "$HOME/Applications/chiaki-ng.AppImage" &> /dev/null rm -rf "$HOME/.config/Chiaki/" &> /dev/null rm -rf "$HOME/.cache/Chiaki/" &> /dev/null rm -f "$romsPath/remoteplay/chiaki-ng.sh" &> /dev/null rm -rf "$HOME/.local/share/applications/chiaki-ng.desktop" &> /dev/null - fi + fi - if [[ "$RPUninstall" != *"Greenlight"* ]]; then + if [[ "$RPUninstall" != *"Greenlight"* ]]; then rm -f "$HOME/Applications/Greenlight.AppImage" &> /dev/null rm -f "$romsPath/remoteplay/Greenlight.sh" &> /dev/null rm -rf "$HOME/.config/greenlight/" &> /dev/null rm -rf "$HOME/.local/share/applications/Greenlight.desktop" &> /dev/null - fi + fi - if [[ "$RPUninstall" != *"Moonlight Game Streaming"* ]]; then + if [[ "$RPUninstall" != *"Moonlight Game Streaming"* ]]; then flatpak uninstall com.moonlight_stream.Moonlight -y rm -f "$romsPath/remoteplay/Moonlight Game Streaming.sh" &> /dev/null rm -rf "$HOME/.var/app/com.moonlight_stream.Moonlight" &> /dev/null - fi + fi - if [[ "$RPUninstall" != *"Parsec"* ]]; then + if [[ "$RPUninstall" != *"Parsec"* ]]; then flatpak uninstall com.parsecgaming.parsec -y rm -f "$romsPath/remoteplay/Parsec.sh" &> /dev/null rm -rf "$HOME/.var/app/com.parsecgaming.parsec" &> /dev/null - fi + fi - if [[ "$RPUninstall" != *"ShadowPC"* ]]; then + if [[ "$RPUninstall" != *"ShadowPC"* ]]; then rm -f "$HOME/Applications/ShadowPC.AppImage" &> /dev/null rm -f "$romsPath/remoteplay/ShadowPC.sh" &> /dev/null rm -rf "$HOME/.config/shadow/" &> /dev/null rm -rf "$HOME/.local/share/applications/ShadowPC.desktop" &> /dev/null - fi + fi - if [[ "$RPUninstall" != *"Steam Link"* ]]; then + if [[ "$RPUninstall" != *"Steam Link"* ]]; then flatpak uninstall com.valvesoftware.SteamLink -y rm -f "$romsPath/remoteplay/SteamLink.sh" &> /dev/null rm -rf "$HOME/.var/app/com.valvesoftware.SteamLink" &> /dev/null - fi + fi + + fi - fi - if find "$romsPath/generic-applications" -type f -name "*.sh" | grep -q .; then GAUninstall=$(zenity --list \ @@ -216,62 +217,62 @@ fi 8 "Tidal" \ 9 "Warehouse" ) - if [[ "$GAUninstall" != *"Bottles"* ]]; then + if [[ "$GAUninstall" != *"Bottles"* ]]; then flatpak uninstall com.usebottles.bottles -y rm -f "$romsPath/generic-applications/Bottles.sh" &> /dev/null rm -rf "$HOME/.var/app/com.usebottles.bottles" &> /dev/null - fi + fi - if [[ "$GAUninstall" != *"Cider"* ]]; then + if [[ "$GAUninstall" != *"Cider"* ]]; then flatpak uninstall sh.cider.Cider -y rm -f "$romsPath/generic-applications/Cider.sh" &> /dev/null rm -rf "$HOME/.var/app/sh.cider.Cider" &> /dev/null - fi + fi - if [[ "$GAUninstall" != *"Flatseal"* ]]; then + if [[ "$GAUninstall" != *"Flatseal"* ]]; then flatpak uninstall com.github.tchx84.Flatseal -y rm -f "$romsPath/generic-applications/Flatseal.sh" &> /dev/null rm -rf "$HOME/.var/app/com.github.tchx84.Flatseal" &> /dev/null - fi + fi if [[ "$GAUninstall" != *"Heroic Games Launcher"* ]]; then rm -f "$HOME/Applications/Heroic-Games-Launcher.AppImage" &> /dev/null rm -f "$romsPath/generic-applications/Heroic-Games-Launcher.sh" &> /dev/null rm -rf "$HOME/.config/heroic/" &> /dev/null rm -rf "$HOME/.local/share/applications/Heroic-Games-Launcher.desktop" &> /dev/null - fi + fi - if [[ "$GAUninstall" != *"Lutris"* ]]; then + if [[ "$GAUninstall" != *"Lutris"* ]]; then flatpak uninstall net.lutris.Lutris -y rm -f "$romsPath/generic-applications/Lutris.sh" &> /dev/null rm -rf "$HOME/.var/app/net.lutris.Lutris" &> /dev/null - fi + fi - if [[ "$GAUninstall" != *"Plexamp"* ]]; then + if [[ "$GAUninstall" != *"Plexamp"* ]]; then flatpak uninstall com.plexamp.Plexamp -y rm -f "$romsPath/generic-applications/Plexamp.sh" &> /dev/null rm -rf "$HOME/.var/app/com.plexamp.Plexamp" &> /dev/null - fi + fi - if [[ "$GAUninstall" != *"Spotify"* ]]; then + if [[ "$GAUninstall" != *"Spotify"* ]]; then flatpak uninstall com.spotify.Client -y rm -f "$romsPath/generic-applications/Spotify.sh" &> /dev/null rm -rf "$HOME/.var/app/com.spotify.Client" &> /dev/null - fi + fi - if [[ "$GAUninstall" != *"Tidal"* ]]; then + if [[ "$GAUninstall" != *"Tidal"* ]]; then flatpak uninstall com.mastermindzh.tidal-hifi -y rm -f "$romsPath/generic-applications/Tidal.sh" &> /dev/null rm -rf "$HOME/.var/app/com.mastermindzh.tidal-hifi" &> /dev/null - fi + fi - if [[ "$GAUninstall" != *"Warehouse"* ]]; then + if [[ "$GAUninstall" != *"Warehouse"* ]]; then flatpak uninstall io.github.flattool.Warehouse -y rm -f "$romsPath/generic-applications/Warehouse.sh" &> /dev/null rm -rf "$HOME/.var/app/io.github.flattool.Warehouse" &> /dev/null - fi + fi - fi + fi #Emulator selector @@ -308,11 +309,12 @@ fi 19 "RPCS3" \ 20 "Ryujinx" \ 21 "ScummVM" \ - 22 "Supermodel" \ - 23 "Vita3K" \ - 24 "Xemu" \ - 25 "Xenia" \ - 26 "Yuzu" ) + 22 "ShadPS4" \ + 23 "Supermodel" \ + 24 "Vita3K" \ + 25 "Xemu" \ + 26 "Xenia" \ + 27 "Yuzu" ) ans=$? @@ -323,7 +325,7 @@ fi fi if [[ "$emusToUninstall" == *"BigPEmu"* ]]; then doUninstallBigPEmu=false - fi + fi if [[ "$emusToUninstall" == *"Cemu"* ]]; then doUninstallCemu=false fi @@ -372,6 +374,9 @@ fi if [[ "$emusToUninstall" == *"RPCS3"* ]]; then doUninstallRPCS3=false fi + if [[ "$emusToUninstall" == *"ShadPS4"* ]]; then + doUninstallShadPS4=false + fi if [[ "$emusToUninstall" == *"RMG"* ]]; then doUninstallRMG=false fi @@ -446,7 +451,7 @@ fi if [[ "$doUninstallFlycast" == true ]]; then flatpak uninstall org.flycast.Flycast -y rm -rf $HOME/.var/app/org.flycast.Flycast &> /dev/null - fi + fi if [[ "$doUninstallLime3DS" == true ]]; then rm -rf $HOME/.config/lime3ds-emu/ &> /dev/null rm -rf $HOME/.local/share/lime3ds-emu &> /dev/null @@ -470,7 +475,7 @@ fi fi if [[ "$doUninstallModel2" == true ]]; then find ${romsPath}/model2 -mindepth 1 -name roms -prune -o -exec rm -rf '{}' \; &>> /dev/null - # Leaving this here for Patreon users stuck with the old name. + # Leaving this here for Patreon users stuck with the old name. rm -f "$HOME/.local/share/applications/Model 2 (Proton).desktop" &> /dev/null # Current name. rm -f "$HOME/.local/share/applications/Model 2 Emulator (Proton).desktop" &> /dev/null @@ -510,6 +515,15 @@ fi rm -rf $HOME/.local/share/applications/RPCS3.desktop &> /dev/null rm -rf $HOME/Applications/rpcs3.AppImage &> /dev/null fi + if [[ "$doUninstallShadPS4" == true ]]; then + # Flatpak + rm -rf $HOME/.config/shadps4 &> /dev/null + # AppImage + rm -rf $HOME/.local/share/shadps4 &> /dev/null + # AppImage + rm -rf $HOME/.local/share/applications/ShadPS4.desktop &> /dev/null + rm -rf $HOME/Applications/shadPS4-qt.AppImage &> /dev/null + fi if [[ "$doUninstallRyujinx" == true ]]; then rm -rf $HOME/.config/Ryujinx &> /dev/null rm -rf $HOME/Applications/publish &> /dev/null @@ -653,21 +667,21 @@ fi # EmulationStation-DE rm -rf $HOME/.emulationstation rm -rf "$HOME/ES-DE" &> /dev/null - rm -rf "$toolsPath/EmulationStation-DE.AppImage" &> /dev/null - rm -rf $HOME/Applications/EmulationStation-DE.AppImage &> /dev/null - rm -rf $HOME/Applications/ES-DE.AppImage &> /dev/null - rm -rf "$HOME/.local/share/applications/EmulationStation-DE.desktop" &> /dev/null - rm -rf "$HOME/.local/share/applications/ES-DE.desktop" &> /dev/null + rm -rf "$toolsPath/EmulationStation-DE.AppImage" &> /dev/null + rm -rf $HOME/Applications/EmulationStation-DE.AppImage &> /dev/null + rm -rf $HOME/Applications/ES-DE.AppImage &> /dev/null + rm -rf "$HOME/.local/share/applications/EmulationStation-DE.desktop" &> /dev/null + rm -rf "$HOME/.local/share/applications/ES-DE.desktop" &> /dev/null # ULWGL rm -rf "$HOME/.local/share/ULWGL" &> /dev/null # SteamDeckGyroDSU rm -rf "$HOME/sdgyrodsu" &> /dev/null # Pegasus flatpak uninstall org.pegasus_frontend.Pegasus -y - rm -rf "$HOME/.var/app/org.pegasus_frontend.Pegasus/" &> /dev/null - rm -rf $HOME/Applications/pegasus-fe &> /dev/null - rm -rf $HOME/.config/pegasus-frontend &> /dev/null - rm -rf "$HOME/.local/share/applications/Pegasus.desktop" &> /dev/null + rm -rf "$HOME/.var/app/org.pegasus_frontend.Pegasus/" &> /dev/null + rm -rf $HOME/Applications/pegasus-fe &> /dev/null + rm -rf $HOME/.config/pegasus-frontend &> /dev/null + rm -rf "$HOME/.local/share/applications/Pegasus.desktop" &> /dev/null echo "90" echo "# Removing EmuDeck folders"; From dcc48832d3260a6242a02f85d08ad9c757cbd62e Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 11:42:06 +0100 Subject: [PATCH 210/230] versions update --- versions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.json b/versions.json index b0f563f31..696b839e1 100644 --- a/versions.json +++ b/versions.json @@ -14,9 +14,9 @@ "shadps4": { "id": "shadps4", "code": "ShadPS4", "version": 0 }, "xemu": { "id": "xemu", "code": "Xemu", "version": 0 }, "cemu": { "id": "cemu", "code": "Cemu", "version": 0 }, - "srm": { "id": "srm", "code": "SRM", "version": 7 }, + "srm": { "id": "srm", "code": "SRM", "version": 8 }, "rmg": { "id": "rmg", "code": "RMG", "version": 0 }, - "esde": { "id": "esde", "code": "ESDE", "version": 3 }, + "esde": { "id": "esde", "code": "ESDE", "version": 4 }, "pegasus": { "id": "pegasus", "code": "Pegasus", "version": 2 }, "mame": { "id": "mame", "code": "MAME", "version": 0 }, "vita3k": { "id": "vita3k", "code": "Vita3k", "version": 0 }, From 13e6d10bdfad9d6fd162793ceace55fc964a46a1 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 11:44:21 +0100 Subject: [PATCH 211/230] sed --- functions/ToolScripts/emuDeckESDE.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/ToolScripts/emuDeckESDE.sh b/functions/ToolScripts/emuDeckESDE.sh index 6429e307e..586528fe8 100644 --- a/functions/ToolScripts/emuDeckESDE.sh +++ b/functions/ToolScripts/emuDeckESDE.sh @@ -116,7 +116,6 @@ ESDE_init(){ rsync -avhp --mkpath "$EMUDECKGIT/chimeraOS/configs/emulationstation/custom_systems/es_find_rules.xml" "$(dirname "$es_rulesFile")" --backup --suffix=.bak # This duplicates ESDE_addCustomSystemsFile but this line only applies only if you are resetting ES-DE and not the emulators themselves. rsync -avhp --mkpath "$EMUDECKGIT/configs/emulationstation/custom_systems/es_systems.xml" "$(dirname "$es_systemsFile")" --backup --suffix=.bak - sed -i "s|/run/media/mmcblk0p1/Emulation|${emulationPath}|g" "$es_rulesFile" ESDE_createLauncher ESDE_addCustomSystems @@ -132,6 +131,8 @@ ESDE_init(){ addSteamInputCustomIcons ESDE_flushToolLauncher SRM_flushOldSymlinks + + sed -i "s|/run/media/mmcblk0p1/Emulation|${emulationPath}|g" "$es_rulesFile" } ESDE_createLauncher(){ From 65ba7391eab5b4237b074e4f8baff32d98d2927a Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Thu, 26 Dec 2024 15:28:43 +0100 Subject: [PATCH 212/230] retro library ps4 support --- tools/retro-library/generate_game_lists.py | 3 +++ tools/retro-library/missing_artwork.py | 2 ++ tools/retro-library/missing_artwork_platforms.py | 2 ++ 3 files changed, 7 insertions(+) diff --git a/tools/retro-library/generate_game_lists.py b/tools/retro-library/generate_game_lists.py index e6337c68b..3e812f807 100644 --- a/tools/retro-library/generate_game_lists.py +++ b/tools/retro-library/generate_game_lists.py @@ -141,6 +141,9 @@ def collect_game_data(system_dir, extensions): system_dir = "xbox360/roms" if system_dir == "model2": system_dir = "model2/roms" + if system_dir == "ps4": + system_dir = "ps4/shortcuts" + full_path = os.path.join(roms_dir, system_dir) if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) diff --git a/tools/retro-library/missing_artwork.py b/tools/retro-library/missing_artwork.py index a792bfeb9..4318e711e 100644 --- a/tools/retro-library/missing_artwork.py +++ b/tools/retro-library/missing_artwork.py @@ -119,6 +119,8 @@ def collect_game_data(system_dir, extensions): system_dir = "xbox360/roms" if system_dir == "model2": system_dir = "model2/roms" + if system_dir == "ps4": + system_dir = "ps4/shortcuts" full_path = os.path.join(roms_dir, system_dir) if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index 5c9552578..be0cb4d97 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -82,6 +82,8 @@ def has_missing_images(system_dir, extensions): system_dir = "xbox360/roms" if system_dir == "model2": system_dir = "model2/roms" + if system_dir == "ps4": + system_dir = "ps4/shortcuts" full_path = os.path.join(roms_dir, system_dir) if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) From cdc52a8919e58b645c81fa04e08639bcbecf450d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 28 Dec 2024 11:42:32 +0100 Subject: [PATCH 213/230] new Ryu url --- functions/EmuScripts/emuDeckRyujinx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/EmuScripts/emuDeckRyujinx.sh b/functions/EmuScripts/emuDeckRyujinx.sh index 4cbfcafe9..97cf350ed 100644 --- a/functions/EmuScripts/emuDeckRyujinx.sh +++ b/functions/EmuScripts/emuDeckRyujinx.sh @@ -49,7 +49,7 @@ Ryujinx_cleanup(){ Ryujinx_install(){ echo "Begin Ryujinx Install" local showProgress=$1 - if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "GreemDev/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then + if installEmuBI "$Ryujinx_emuName" "$(getReleaseURLGH "Ryubing/Ryujinx" "-linux_x64.tar.gz")" "" "tar.gz" "$showProgress"; then mkdir -p "$HOME/Applications/publish" tar -xvf "$HOME/Applications/Ryujinx.tar.gz" -C "$HOME/Applications" && rm -rf "$HOME/Applications/Ryujinx.tar.gz" chmod +x "$HOME/Applications/publish/Ryujinx" From 7636fc3729f79df1fb56266f7f4ad78bc8fbbe9b Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 28 Dec 2024 17:22:48 +0100 Subject: [PATCH 214/230] nohash --- functions/generateGameLists.sh | 4 +- tools/retro-library/missing_artwork_nohash.py | 168 ++++++++++++++++++ 2 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 tools/retro-library/missing_artwork_nohash.py diff --git a/functions/generateGameLists.sh b/functions/generateGameLists.sh index 7104f5d64..0bb7533f2 100644 --- a/functions/generateGameLists.sh +++ b/functions/generateGameLists.sh @@ -91,10 +91,10 @@ generateGameLists_getPercentage() { local accountfolder=$(ls -td $HOME/.steam/steam/userdata/* | head -n 1) local dest_folder="$storagePath/retrolibrary/artwork/" - #python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork.py "$romsPath" "$dest_folder" + python $HOME/.config/EmuDeck/backend/tools/retro-library/missing_artwork_nohash.py "$romsPath" "$dest_folder" local json_file="$storagePath/retrolibrary/cache/roms_games.json" - local json_file_artwork="$storagePath/retrolibrary/cache/missing_artwork.json" + local json_file_artwork="$storagePath/retrolibrary/cache/missing_artwork_no_hash.json" # Contar el número total de juegos en `roms_games.json` local games=$(jq '[.[].games[]] | length' "$json_file") diff --git a/tools/retro-library/missing_artwork_nohash.py b/tools/retro-library/missing_artwork_nohash.py new file mode 100644 index 000000000..c960d9a11 --- /dev/null +++ b/tools/retro-library/missing_artwork_nohash.py @@ -0,0 +1,168 @@ +import os +import json +import sys +import re +import subprocess +import hashlib + +# Define the log file path +home_dir = os.environ.get("HOME") +msg_file = os.path.join(home_dir, ".config/EmuDeck/msg.log") + +def getSettings(): + pattern = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)=(.*)') + user_home = os.path.expanduser("~") + + if os.name == 'nt': + config_file_path = os.path.join(user_home, 'emudeck', 'settings.ps1') + else: + config_file_path = os.path.join(user_home, 'emudeck', 'settings.sh') + + configuration = {} + + with open(config_file_path, 'r') as file: + for line in file: + match = pattern.search(line) + if match: + variable = match.group(1) + value = match.group(2).strip().strip('"') + expanded_value = os.path.expandvars(value.replace('"', '').replace("'", "")) + configuration[variable] = expanded_value + + # Obtener rama actual del repositorio backend + if os.name == 'nt': + bash_command = f"cd {os.path.join(user_home, 'AppData', 'Roaming', 'EmuDeck', 'backend')} && git rev-parse --abbrev-ref HEAD" + else: + bash_command = "cd $HOME/.config/EmuDeck/backend/ && git rev-parse --abbrev-ref HEAD" + + result = subprocess.run(bash_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + configuration["branch"] = result.stdout.strip() + + configuration["systemOS"] = os.name + + return configuration + +settings = getSettings() +storage_path = os.path.expandvars(settings["storagePath"]) + +# Function to write messages to the log file +def log_message(message): + with open(msg_file, "w") as log_file: # "a" to append messages without overwriting + log_file.write(message + "\n") + +def generate_game_lists(roms_path, images_path): + def calculate_hash(file_path): + import hashlib + hash_md5 = hashlib.md5() + try: + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(65536), b""): # 64 KB chunks + hash_md5.update(chunk) + print(file_path , hash_md5.hexdigest()) + return hash_md5.hexdigest() + except Exception: + return None + + def collect_game_data(system_dir, extensions): + game_data = [] + for root, _, files in os.walk(system_dir): + for file in files: + file_path = os.path.join(root, file) + if os.path.islink(file_path): + continue + + filename = os.path.basename(file) + extension = filename.split('.')[-1] + name = '.'.join(filename.split('.')[:-1]) + if extension in extensions: + # Special cases for WiiU and PS3 + if "wiiu" in system_dir: + parts = root.split(os.sep) + name = parts[-2] if len(parts) >= 2 else name + if "ps3" in system_dir: + parts = root.split(os.sep) + name = parts[-3] if len(parts) >= 3 else name + + platform = os.path.basename(system_dir) + + # Clean the game name + name_cleaned = re.sub(r'\(.*?\)', '', name) + name_cleaned = re.sub(r'\[.*?\]', '', name_cleaned) + name_cleaned = name_cleaned.strip().replace(' ', '_').replace('-', '_') + name_cleaned = re.sub(r'_+', '_', name_cleaned) + name_cleaned = name_cleaned.replace('+', '').replace('&', '').replace('!', '').replace("'", '').replace('.', '') + name_cleaned_pegasus = name.replace(',_', ',') + name_cleaned = name_cleaned.lower() + + # Check for missing images + for img_type, ext in [("box2dfront", ".jpg"), ("wheel", ".png"), ("screenshot", ".jpg")]: + img_path = os.path.join(images_path, f"{platform}/media/{img_type}/{name_cleaned}{ext}") + if not os.path.exists(img_path): + log_message(f"Missing image: {img_path}") + + game_info = { + "name": name_cleaned, + "platform": platform, + "type": img_type + } + game_data.append(game_info) + + return sorted(game_data, key=lambda x: x['name']) + + roms_dir = roms_path + valid_system_dirs = [] + + for system_dir in os.listdir(roms_dir): + if system_dir == "xbox360": + system_dir = "xbox360/roms" + if system_dir == "model2": + system_dir = "model2/roms" + if system_dir == "ps4": + system_dir = "ps4/shortcuts" + full_path = os.path.join(roms_dir, system_dir) + if os.path.isdir(full_path) and not os.path.islink(full_path) and os.path.isfile(os.path.join(full_path, 'metadata.txt')): + file_count = sum([len(files) for r, d, files in os.walk(full_path) if not os.path.islink(r)]) + if file_count > 2: + valid_system_dirs.append(full_path) + log_message(f"MA: Valid system directory found: {full_path}") + + game_list = [] + + for system_dir in valid_system_dirs: + if any(x in system_dir for x in ["/model2", "/genesiswide", "/mame", "/emulators", "/desktop"]): + log_message(f"MA: Skipping directory: {system_dir}") + continue + + with open(os.path.join(system_dir, 'metadata.txt')) as f: + metadata = f.read() + collection = next((line.split(':')[1].strip() for line in metadata.splitlines() if line.startswith('collection:')), '') + shortname = next((line.split(':')[1].strip() for line in metadata.splitlines() if line.startswith('shortname:')), '') + launcher = next((line.split(':', 1)[1].strip() for line in metadata.splitlines() if line.startswith('launch:')), '').replace('"', '\\"') + extensions = next((line.split(':')[1].strip().replace(',', ' ') for line in metadata.splitlines() if line.startswith('extensions:')), '').split() + + games = collect_game_data(system_dir, extensions) + if games: + system_info = { + "title": collection, + "id": shortname, + "launcher": launcher, + "games": games + } + game_list.append(system_info) + log_message(f"MA: Detected {len(games)} games from {system_dir}") + + json_output = json.dumps(sorted(game_list, key=lambda x: x['title']), indent=4) + + output_file = os.path.join(storage_path, "retrolibrary/cache/missing_artwork_no_hash.json") + + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'w') as f: + f.write(json_output) + #print(json_output) + +roms_path = sys.argv[1] +images_path = sys.argv[2] + +log_message("MA: Missing artwork list generation in progress...") +generate_game_lists(roms_path, images_path) +log_message("MA: Missing artwork list process completed.") \ No newline at end of file From e387038d92c560100ac0abc0a10558b433832a2d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 28 Dec 2024 17:32:46 +0100 Subject: [PATCH 215/230] bad indent --- tools/retro-library/missing_artwork_platforms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/retro-library/missing_artwork_platforms.py b/tools/retro-library/missing_artwork_platforms.py index be0cb4d97..7e1faee57 100644 --- a/tools/retro-library/missing_artwork_platforms.py +++ b/tools/retro-library/missing_artwork_platforms.py @@ -79,7 +79,7 @@ def has_missing_images(system_dir, extensions): for system_dir in os.listdir(roms_dir): if system_dir == "xbox360": - system_dir = "xbox360/roms" + system_dir = "xbox360/roms" if system_dir == "model2": system_dir = "model2/roms" if system_dir == "ps4": From df7c35ce3adacef79f822f00bc55a5da3c4fb814 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 28 Dec 2024 17:36:14 +0100 Subject: [PATCH 216/230] old zip --- functions/ToolScripts/emuDeckPlugins.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/ToolScripts/emuDeckPlugins.sh b/functions/ToolScripts/emuDeckPlugins.sh index 027e1be99..0a9891991 100644 --- a/functions/ToolScripts/emuDeckPlugins.sh +++ b/functions/ToolScripts/emuDeckPlugins.sh @@ -134,21 +134,21 @@ Plugins_installDeckyRomLibrary(){ echo "Installing Decky Rom Library" local password=$1 local destinationFolder="$HOME/homebrew/plugins/decky-rom-library" - local DeckyControls_releaseURL="$(getLatestReleaseURLGH "EmuDeck/decky-rom-library" ".7z")" + local DeckyControls_releaseURL="$(getLatestReleaseURLGH "EmuDeck/decky-rom-library" ".zip")" mkdir -p "$HOME/homebrew/plugins/" Plugins_installPluginLoader $password if [ -d "$HOME/homebrew" ]; then Plugins_checkPassword $password echo $password | sudo -S rm -rf $destinationFolder - echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.7z" - echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.7z" -d "$HOME/homebrew/plugins/" && echo $password | sudo -S rm "$HOME/homebrew/plugins/decky-rom-library.7z" + echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.zip" + echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.zip" -d "$HOME/homebrew/plugins/" && echo $password | sudo -S rm "$HOME/homebrew/plugins/decky-rom-library.zip" echo $password | sudo -S chown $USER:$USER -R $HOME/homebrew/plugins/decky-rom-library echo $password | sudo -S chmod 555 -R $HOME/homebrew/plugins/decky-rom-library Plugins_install_cleanup $password else rm -rf $destinationFolder - echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.7z" - echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.7z" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/decky-rom-library.7z" + echo $password | sudo -S curl -L "$DeckyControls_releaseURL" -o "$HOME/homebrew/plugins/decky-rom-library.zip" + echo $password | sudo -S unzip "$HOME/homebrew/plugins/decky-rom-library.zip" -d "$HOME/homebrew/plugins/" && sudo rm "$HOME/homebrew/plugins/decky-rom-library.zip" echo $password | sudo -S chown $USER:$USER -R $HOME/homebrew/plugins/decky-rom-library fi #RAachievemets From 04bfdd423e05a625486590f22479875d09116946 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sat, 28 Dec 2024 23:34:55 +0100 Subject: [PATCH 217/230] cloudsync and netplay --- functions/helperFunctions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/helperFunctions.sh b/functions/helperFunctions.sh index e6b854e45..deb38b9cd 100644 --- a/functions/helperFunctions.sh +++ b/functions/helperFunctions.sh @@ -1045,7 +1045,7 @@ function emulatorInit(){ #Check if the service is up and running - if [ -f "$cloud_sync_bin" ] && [ "$cloud_sync_status" == "true" ]; then + if [ -f "$cloud_sync_bin" ] && [ "$cloud_sync_status" == "true" ] && [ "$netPlay" != "true" ]; then if [ $(check_internet_connection) == "true" ]; then From ccbe279eb0f3b596d81c1f149cf4e755b104e34d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 19:37:08 +0100 Subject: [PATCH 218/230] & --- install-early.sh | 2 +- install-unstable.sh | 2 +- install.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/install-early.sh b/install-early.sh index a5632e0f0..58b6b9437 100644 --- a/install-early.sh +++ b/install-early.sh @@ -109,5 +109,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox +~/Applications/EmuDeck.AppImage $sandbox & exit diff --git a/install-unstable.sh b/install-unstable.sh index 21ee999ab..26542bc51 100644 --- a/install-unstable.sh +++ b/install-unstable.sh @@ -109,5 +109,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox +~/Applications/EmuDeck.AppImage $sandbox & exit diff --git a/install.sh b/install.sh index d5d3b6635..670ad6ab0 100644 --- a/install.sh +++ b/install.sh @@ -109,5 +109,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox +~/Applications/EmuDeck.AppImage $sandbox & exit From 3391c61b507967b599fbe34b90232b305a33387d Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:01:09 +0100 Subject: [PATCH 219/230] timed setup --- setup.sh | 160 +++++++++++++++++++++++++++---------------------------- 1 file changed, 78 insertions(+), 82 deletions(-) diff --git a/setup.sh b/setup.sh index f27d5c8db..a38e56a15 100644 --- a/setup.sh +++ b/setup.sh @@ -99,7 +99,7 @@ SECONDTIME="$HOME/.config/EmuDeck/.finished" #Lets log github API limits just in case echo 'Github API limits:' -curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/rate_limit" +time curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/rate_limit" # ## @@ -119,7 +119,7 @@ EMUDECKGIT="$HOME/.config/EmuDeck/backend" source "$EMUDECKGIT"/functions/helperFunctions.sh source "$EMUDECKGIT"/functions/jsonToBashVars.sh -jsonToBashVars "$HOME/.config/EmuDeck/settings.json" +time jsonToBashVars "$HOME/.config/EmuDeck/settings.json" source "$EMUDECKGIT/functions/all.sh" @@ -137,8 +137,8 @@ grep -vi pass "$emuDecksettingsFile" # echo "" echo "Env Details: " -getEnvironmentDetails -testRealDeck +time getEnvironmentDetails +time testRealDeck #this sets up the settings file with defaults, in case they don't have a new setting we've added. #also echos them all out so they are in the log. @@ -146,7 +146,7 @@ testRealDeck #createUpdateSettingsFile #create folders after tests! -createFolders +time createFolders #setup Proton-Launch.sh #because this path gets updated by sed, we really should be installing it every time and allowing it to be updated every time. In case the user changes their path. @@ -161,152 +161,152 @@ chmod +x "${toolsPath}/emu-launch.sh" #ESDE Installation if [ $doInstallESDE == "true" ]; then echo "install esde" - ESDE_install + time ESDE_install fi #Pegasus Installation if [ $doInstallPegasus == "true" ]; then echo "install Pegasus" - pegasus_install + time pegasus_install fi #SRM Installation if [ $doInstallSRM == "true" ]; then echo "install srm" - SRM_install + time SRM_install fi if [ "$doInstallPCSX2QT" == "true" ]; then echo "install pcsx2Qt" - PCSX2QT_install + time PCSX2QT_install fi if [ $doInstallPrimeHack == "true" ]; then echo "install primehack" - Primehack_install + time Primehack_install fi if [ $doInstallRPCS3 == "true" ]; then echo "install rpcs3" - RPCS3_install + time RPCS3_install fi if [ $doInstallCitra == "true" ]; then echo "install Citra" - Citra_install + time Citra_install fi if [ $doInstallLime3DS == "true" ]; then echo "install Lime3DS" - Lime3DS_install + time Lime3DS_install fi if [ $doInstallDolphin == "true" ]; then echo "install Dolphin" - Dolphin_install + time Dolphin_install fi if [ $doInstallDuck == "true" ]; then echo "DuckStation_install" - DuckStation_install + time DuckStation_install fi if [ $doInstallRA == "true" ]; then echo "RetroArch_install" - RetroArch_install + time RetroArch_install fi if [ $doInstallRMG == "true" ]; then echo "RMG_install" - RMG_install + time RMG_install fi if [ $doInstallares == "true" ]; then echo "ares_install" - ares_install + time ares_install fi if [ $doInstallPPSSPP == "true" ]; then echo "PPSSPP_install" - PPSSPP_install + time PPSSPP_install fi if [ $doInstallYuzu == "true" ]; then echo "Yuzu_install" - Yuzu_install + time Yuzu_install fi if [ $doInstallSuyu == "true" ]; then echo "suyu_install" - suyu_install + time suyu_install fi if [ $doInstallRyujinx == "true" ]; then echo "Ryujinx_install" - Ryujinx_install + time Ryujinx_install fi if [ $doInstallMAME == "true" ]; then echo "MAME_install" - MAME_install + time MAME_install fi if [ $doInstallXemu == "true" ]; then echo "Xemu_install" - Xemu_install + time Xemu_install fi if [ $doInstallCemu == "true" ]; then echo "Cemu_install" - Cemu_install + time Cemu_install fi if [ "${doInstallCemuNative}" == "true" ]; then echo "CemuNative_install" - CemuNative_install + time CemuNative_install fi if [ $doInstallScummVM == "true" ]; then echo "ScummVM_install" - ScummVM_install + time ScummVM_install fi if [ $doInstallVita3K == "true" ]; then echo "Vita3K_install" - Vita3K_install + time Vita3K_install fi if [ $doInstallMGBA == "true" ]; then echo "mGBA_install" - mGBA_install + time mGBA_install fi if [ $doInstallFlycast == "true" ]; then echo "Flycast_install" - Flycast_install + time Flycast_install fi if [ $doInstallRMG == "true" ]; then echo "RMG_install" - RMG_install + time RMG_install fi if [ $doInstallares == "true" ]; then echo "ares_install" - ares_install + time ares_install fi if [ $doInstallmelonDS == "true" ]; then echo "melonDS_install" - melonDS_install + time melonDS_install fi if [ $doInstallBigPEmu == "true" ]; then echo "BigPEmu_install" - BigPEmu_install + time BigPEmu_install fi if [ $doInstallSupermodel == "true" ]; then echo "Supermodel_install" - Supermodel_install + time Supermodel_install fi #Xenia - We need to install Xenia after creating the Roms folders! if [ "$doInstallXenia" == "true" ]; then echo "Xenia_install" - Xenia_install + time Xenia_install fi if [ "$doInstallModel2" == "true" ]; then echo "Model2_install" - Model2_install + time Model2_install fi if [ "$doInstallShadPS4" == "true" ]; then echo "ShadPS4_install" - ShadPS4_install + time ShadPS4_install fi #Steam RomManager Config if [ "$doSetupSRM" == "true" ]; then echo "SRM_init" - SRM_init + time SRM_init fi #ESDE Config if [ "$doSetupESDE" == "true" ]; then echo "ESDE_init" - ESDE_update + time ESDE_update fi #Pegasus Config @@ -323,116 +323,112 @@ setMSG "Configuring emulators.." if [ "$doSetupRA" == "true" ]; then echo "RetroArch_init" - RetroArch_init + time RetroArch_init fi if [ "$doSetupPrimehack" == "true" ]; then echo "Primehack_init" - Primehack_init + time Primehack_init fi if [ "$doSetupDolphin" == "true" ]; then echo "Dolphin_init" - Dolphin_init + time Dolphin_init fi if [ "$doSetupPCSX2QT" == "true" ]; then echo "PCSX2QT_init" - PCSX2QT_init + time PCSX2QT_init fi if [ "$doSetupRPCS3" == "true" ]; then echo "RPCS3_init" - RPCS3_init + time RPCS3_init fi if [ "$doSetupCitra" == "true" ]; then echo "Citra_init" - Citra_init + time Citra_init fi if [ $doSetupLime3DS == "true" ]; then echo "Lime3DS_init" - Lime3DS_init + time Lime3DS_init fi if [ "$doSetupDuck" == "true" ]; then echo "DuckStation_init" - DuckStation_init + time DuckStation_init fi if [ "$doSetupYuzu" == "true" ]; then echo "Yuzu_init" - Yuzu_init -fi -if [ "$doSetupSuyu" == "true" ]; then - echo "suzu_init" - suzu_init + time Yuzu_init fi if [ "$doSetupRyujinx" == "true" ]; then echo "Ryujinx_init" - Ryujinx_init + time Ryujinx_init fi if [ "$doSetupShadPS4" == "true" ]; then echo "ShadPS4_init" - ShadPS4_init + time ShadPS4_init fi if [ "$doSetupPPSSPP" == "true" ]; then echo "PPSSPP_init" - PPSSPP_init + time PPSSPP_init fi if [ "$doSetupXemu" == "true" ]; then echo "Xemu_init" - Xemu_init + time Xemu_init fi if [ "$doSetupMAME" == "true" ]; then echo "MAME_init" - MAME_init + time MAME_init fi if [ "$doSetupScummVM" == "true" ]; then echo "ScummVM_init" - ScummVM_init + time ScummVM_init fi if [ "$doSetupVita3K" == "true" ]; then echo "Vita3K_init" - Vita3K_init + time Vita3K_init fi if [ "$doSetupRMG" == "true" ]; then echo "RMG_init" - RMG_init + time RMG_init fi if [ "$doSetupares" == "true" ]; then echo "ares_init" - ares_init + time ares_init fi if [ "$doSetupmelonDS" == "true" ]; then echo "melonDS_init" - melonDS_init + time melonDS_init fi if [ "$doSetupMGBA" == "true" ]; then echo "mGBA_init" - mGBA_init + time mGBA_init fi if [ "${doSetupCemuNative}" == "true" ]; then echo "CemuNative_init" - CemuNative_init + time CemuNative_init fi if [ "$doSetupFlycast" == "true" ]; then echo "Flycast_init" - Flycast_init + time Flycast_init fi if [ "$doSetupSupermodel" == "true" ]; then echo "Supermodel_init" - Supermodel_init + time Supermodel_init fi if [ "$doSetupModel2" == "true" ]; then echo "model2_init" - Model2_init + time Model2_init fi #Proton Emus if [ "$doSetupCemu" == "true" ]; then echo "Cemu_init" - Cemu_init + time Cemu_init fi if [ "$doSetupBigPEmu" == "true" ]; then echo "BigPEmu_init" - BigPEmu_init + time BigPEmu_init fi if [ "$doSetupXenia" == "true" ]; then echo "Xenia_init" - Xenia_init + time Xenia_init fi @@ -444,11 +440,11 @@ fi #Always install -BINUP_install -AutoCopy_install -server_install -FlatpakUP_install -CHD_install +time BINUP_install +time AutoCopy_install +time server_install +time FlatpakUP_install +time CHD_install # ## @@ -462,12 +458,12 @@ CHD_install # if [ "$doSetupRA" == "true" ]; then if [ "$(getScreenAR)" == 169 ];then - nonDeck_169Screen + time nonDeck_169Screen fi #Anbernic Win600 Special configuration if [ "$(getProductName)" == "Win600" ];then - nonDeck_win600 + time nonDeck_win600 fi fi @@ -487,13 +483,13 @@ if [ "$system" == "chimeraos" ]; then fi -createDesktopIcons +time createDesktopIcons if [ "$controllerLayout" == "bayx" ] || [ "$controllerLayout" == "baxy" ] ; then - controllerLayout_BAYX + time controllerLayout_BAYX else - controllerLayout_ABXY + time controllerLayout_ABXY fi # @@ -529,7 +525,7 @@ rm "$PIDFILE" ## We check all the selected emulators are installed # -checkInstalledEmus +time checkInstalledEmus # From e32d4f54cb6de287a9985f7a8f995da3400112c0 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:04:09 +0100 Subject: [PATCH 220/230] test --- install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/install.sh b/install.sh index 670ad6ab0..5b8b64087 100644 --- a/install.sh +++ b/install.sh @@ -110,4 +110,3 @@ mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage ~/Applications/EmuDeck.AppImage $sandbox & -exit From f0439ba76e98436efee4046d2b62be70c659f477 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:06:00 +0100 Subject: [PATCH 221/230] nohup --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 5b8b64087..e44e301b1 100644 --- a/install.sh +++ b/install.sh @@ -109,4 +109,4 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox & +nohup ~/Applications/EmuDeck.AppImage $sandbox & From 02ec122de1eacf2264e64a568cdd523b75a5a870 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:07:23 +0100 Subject: [PATCH 222/230] test --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index e44e301b1..e7cd8d1f3 100644 --- a/install.sh +++ b/install.sh @@ -110,3 +110,4 @@ mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage nohup ~/Applications/EmuDeck.AppImage $sandbox & +sleep 10 \ No newline at end of file From 8bdae431d48628e5be5b368abe7bbbd402eae6c4 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:09:12 +0100 Subject: [PATCH 223/230] test --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index e7cd8d1f3..f46fa4b93 100644 --- a/install.sh +++ b/install.sh @@ -109,5 +109,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -nohup ~/Applications/EmuDeck.AppImage $sandbox & +nohup ~/Applications/EmuDeck.AppImage $sandbox > /dev/null 2>&1 & sleep 10 \ No newline at end of file From 8aba971d1b8e5cb7c6e5721c0161aaceac3c08c5 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:12:12 +0100 Subject: [PATCH 224/230] test --- install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install.sh b/install.sh index f46fa4b93..5179e71c4 100644 --- a/install.sh +++ b/install.sh @@ -109,5 +109,4 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -nohup ~/Applications/EmuDeck.AppImage $sandbox > /dev/null 2>&1 & -sleep 10 \ No newline at end of file +~/Applications/EmuDeck.AppImage $sandbox > /dev/null 2>&1 & disown \ No newline at end of file From 7c8e7d9e00898bfbbf9ce72875449d242caddaaf Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:14:01 +0100 Subject: [PATCH 225/230] test --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 5179e71c4..274a54496 100644 --- a/install.sh +++ b/install.sh @@ -109,4 +109,4 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox > /dev/null 2>&1 & disown \ No newline at end of file +setsid ~/Applications/EmuDeck.AppImage $sandbox & \ No newline at end of file From 827724ddc233f922a61a3c8fdc9f7493d5f26407 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:15:12 +0100 Subject: [PATCH 226/230] test --- install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 274a54496..1cf7d4a8d 100644 --- a/install.sh +++ b/install.sh @@ -6,9 +6,9 @@ sandbox="" if [ $linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi - +clear if [ "$linuxID" == "SteamOS" ]; then - echo "installing EmuDeck" + echo "Installing EmuDeck" else zenityAvailable=$(command -v zenity &> /dev/null && echo true) @@ -109,4 +109,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage +echo "setsid" setsid ~/Applications/EmuDeck.AppImage $sandbox & \ No newline at end of file From 224aaf5bb36be522d5ba1e63154c1a3c7c5a94ba Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:17:00 +0100 Subject: [PATCH 227/230] back to normal --- install-early.sh | 2 +- install-unstable.sh | 2 +- install.sh | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/install-early.sh b/install-early.sh index 58b6b9437..a5632e0f0 100644 --- a/install-early.sh +++ b/install-early.sh @@ -109,5 +109,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox & +~/Applications/EmuDeck.AppImage $sandbox exit diff --git a/install-unstable.sh b/install-unstable.sh index 26542bc51..21ee999ab 100644 --- a/install-unstable.sh +++ b/install-unstable.sh @@ -109,5 +109,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox & +~/Applications/EmuDeck.AppImage $sandbox exit diff --git a/install.sh b/install.sh index 1cf7d4a8d..bf225a055 100644 --- a/install.sh +++ b/install.sh @@ -109,5 +109,4 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -echo "setsid" -setsid ~/Applications/EmuDeck.AppImage $sandbox & \ No newline at end of file +~/Applications/EmuDeck.AppImage $sandbox \ No newline at end of file From e460f01ad0c2d04955fa52fb50a90ab61ba57f91 Mon Sep 17 00:00:00 2001 From: Dragoon Dorise Date: Sun, 29 Dec 2024 20:43:55 +0100 Subject: [PATCH 228/230] bye dolphin & citra zenity --- functions/EmuScripts/emuDeckCitra.sh | 10 ----- functions/EmuScripts/emuDeckDolphin.sh | 53 ++++++++++---------------- 2 files changed, 21 insertions(+), 42 deletions(-) diff --git a/functions/EmuScripts/emuDeckCitra.sh b/functions/EmuScripts/emuDeckCitra.sh index 0d10d0306..e7dda34f8 100755 --- a/functions/EmuScripts/emuDeckCitra.sh +++ b/functions/EmuScripts/emuDeckCitra.sh @@ -304,11 +304,6 @@ Citra_flushSymlinks(){ find "$STEAMPATH/userdata" -name "shortcuts.vdf" -exec sed -i "s|${romsPath}/3ds|${romsPath}/n3ds|g" {} + touch "$HOME/.config/EmuDeck/.citralegacysymlinks" echo "Citra symlink cleanup completed." - zenity --info \ - --text="Citra symlinks have been cleaned. This cleanup was conducted to prevent any potential breakage with symlinks. Place all new ROMs in Emulation/roms/n3ds. Your ROMs have been moved from Emulation/roms/3ds to Emulation/roms/n3ds." \ - --title="Symlink Update" \ - --width=400 \ - --height=300 else echo "Citra symlinks already cleaned." @@ -366,11 +361,6 @@ Citra_flushSymlinks(){ find "$STEAMPATH/userdata" -name "shortcuts.vdf" -exec sed -i "s|${romsPath}/3ds|${romsPath}/n3ds|g" {} + touch "$HOME/.config/EmuDeck/.citrasymlinks" echo "Citra symlink cleanup completed." - zenity --info \ - --text="Citra symlinks have been cleaned. This cleanup was conducted to prevent any potential breakage with symlinks. Place all new ROMs in Emulation/roms/n3ds. Your ROMs have been moved from Emulation/roms/3ds to Emulation/roms/n3ds." \ - --title="Symlink Update" \ - --width=400 \ - --height=300 else echo "Citra symlinks already cleaned." diff --git a/functions/EmuScripts/emuDeckDolphin.sh b/functions/EmuScripts/emuDeckDolphin.sh index 1813ddcf1..4f2d5faaf 100644 --- a/functions/EmuScripts/emuDeckDolphin.sh +++ b/functions/EmuScripts/emuDeckDolphin.sh @@ -41,7 +41,7 @@ Dolphin_install(){ #ApplyInitialSettings Dolphin_init(){ - + setMSG "${Dolphin_emuName}: Apply initial config" echo "" configEmuFP "${Dolphin_emuName}" "${Dolphin_emuPath}" "true" @@ -243,14 +243,14 @@ Dolphin_flushSymlinks(){ rm -rf "$romsPath/gc/metadata.txt" &> /dev/null rm -rf "$romsPath/gc/systeminfo.txt" &> /dev/null - # The Pegasus install was accidentally overwriting the pre-existing n3ds symlink. - # This checks if the gc folder is empty (post-removing the contents above) and if the original gamecube folder is still a folder and not a symlink (for those who have already migrated). - # If all of this is true, the gc folder is deleted and the old symlink is temporarily recreated to proceed with the migration. + # The Pegasus install was accidentally overwriting the pre-existing n3ds symlink. + # This checks if the gc folder is empty (post-removing the contents above) and if the original gamecube folder is still a folder and not a symlink (for those who have already migrated). + # If all of this is true, the gc folder is deleted and the old symlink is temporarily recreated to proceed with the migration. if [[ ! "$( ls -A "$romsPath/gc")" ]] && [ -d "$romsPath/gamecube" ] && [ ! -L "$romsPath/gamecube" ]; then rm -rf "$romsPath/gc" - ln -sfn "$romsPath/gamecube" "$romsPath/gc" - # Temporarily restores old directory structure. - fi + ln -sfn "$romsPath/gamecube" "$romsPath/gc" + # Temporarily restores old directory structure. + fi if [[ -L "$romsPath/gc" && ! $(readlink -f "$romsPath/gc") =~ ^"$romsPath" ]] || [[ -L "$romsPath/gamecube" && ! $(readlink -f "$romsPath/gamecube") =~ ^"$romsPath" ]]; then echo "User has symlinks that don't match expected paths located under $romsPath. Aborting symlink update." @@ -278,22 +278,16 @@ Dolphin_flushSymlinks(){ rsync -avh "$EMUDECKGIT/roms/gc/." "$romsPath/gc/." --ignore-existing - if [ -d "$toolsPath/downloaded_media/gc" ] && [ ! -d "$romsPath/gc/media" ]; then + if [ -d "$toolsPath/downloaded_media/gc" ] && [ ! -d "$romsPath/gc/media" ]; then ln -s "$toolsPath/downloaded_media/gc" "$romsPath/gc/media" fi find "$STEAMPATH/userdata" -name "shortcuts.vdf" -exec sed -i "s|${romsPath}/gamecube|${romsPath}/gc|g" {} + - touch "$HOME/.config/EmuDeck/.dolphinlegacysymlinks" + touch "$HOME/.config/EmuDeck/.dolphinlegacysymlinks" echo "Dolphin symlink cleanup completed." - zenity --info \ - --text="Dolphin symlinks have been cleaned. This cleanup was conducted to prevent any potential breakage with symlinks. Place all new ROMs in Emulation/roms/gc. Your ROMs have been moved from Emulation/roms/gamecube to Emulation/roms/gc." \ - --title="Symlink Update" \ - --width=400 \ - --height=300 - - else + else echo "Dolphin symlinks already cleaned." - fi + fi if [ ! -f "$HOME/.config/EmuDeck/.dolphinsymlinks" ]; then @@ -303,14 +297,14 @@ Dolphin_flushSymlinks(){ rm -rf "$romsPath/gc/metadata.txt" &> /dev/null rm -rf "$romsPath/gc/systeminfo.txt" &> /dev/null - # The Pegasus install was accidentally overwriting the pre-existing n3ds symlink. - # This checks if the gc folder is empty (post-removing the contents above) and if the original gamecube folder is still a folder and not a symlink (for those who have already migrated). - # If all of this is true, the gc folder is deleted and the old symlink is temporarily recreated to proceed with the migration. + # The Pegasus install was accidentally overwriting the pre-existing n3ds symlink. + # This checks if the gc folder is empty (post-removing the contents above) and if the original gamecube folder is still a folder and not a symlink (for those who have already migrated). + # If all of this is true, the gc folder is deleted and the old symlink is temporarily recreated to proceed with the migration. if [[ ! "$( ls -A "$romsPath/gc")" ]] && [ -d "$romsPath/gamecube" ] && [ ! -L "$romsPath/gamecube" ]; then rm -rf "$romsPath/gc" - ln -sfn "$romsPath/gamecube" "$romsPath/gc" - # Temporarily restores old directory structure. - fi + ln -sfn "$romsPath/gamecube" "$romsPath/gc" + # Temporarily restores old directory structure. + fi if [[ -L "$romsPath/gc" && ! $(readlink -f "$romsPath/gc") =~ ^"$romsPath" ]] || [[ -L "$romsPath/gamecube" && ! $(readlink -f "$romsPath/gamecube") =~ ^"$romsPath" ]]; then echo "User has symlinks that don't match expected paths located under $romsPath. Aborting symlink update." @@ -338,21 +332,16 @@ Dolphin_flushSymlinks(){ rsync -avh "$EMUDECKGIT/roms/gc/." "$romsPath/gc/." --ignore-existing - if [ -d "$toolsPath/downloaded_media/gc" ] && [ ! -d "$romsPath/gc/media" ]; then + if [ -d "$toolsPath/downloaded_media/gc" ] && [ ! -d "$romsPath/gc/media" ]; then ln -s "$toolsPath/downloaded_media/gc" "$romsPath/gc/media" fi find "$STEAMPATH/userdata" -name "shortcuts.vdf" -exec sed -i "s|${romsPath}/gamecube|${romsPath}/gc|g" {} + - touch "$HOME/.config/EmuDeck/.dolphinsymlinks" + touch "$HOME/.config/EmuDeck/.dolphinsymlinks" echo "Dolphin symlink cleanup completed." - zenity --info \ - --text="Dolphin symlinks have been cleaned. This cleanup was conducted to prevent any potential breakage with symlinks. Place all new ROMs in Emulation/roms/gc. Your ROMs have been moved from Emulation/roms/gamecube to Emulation/roms/gc." \ - --title="Symlink Update" \ - --width=400 \ - --height=300 - else + else echo "Dolphin symlinks already cleaned." - fi + fi } \ No newline at end of file From 9b967cd3a8182a56fa3cb4f40ca86f99701fe916 Mon Sep 17 00:00:00 2001 From: marianofalzone <140389938+marianofalzone@users.noreply.github.com> Date: Mon, 30 Dec 2024 06:46:36 +0000 Subject: [PATCH 229/230] Added json for Petrophobia (#1373) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixes dolphin control issues * Revert "fixes dolphin control issues" This reverts commit 32945b947a0306418d563386cd6c0e17030cf6c4. * dolphin for real * versions * forced citra * Fix Cemu - No DSU * GreemDev * path fix * Update emuDeckXenia.sh * Update install.sh (#1369) * fixes dolphin control issues * Revert "fixes dolphin control issues" This reverts commit 32945b947a0306418d563386cd6c0e17030cf6c4. * dolphin for real * versions * forced citra * Fix Cemu - No DSU * GreemDev * path fix * Update install.sh On Ubuntu 24.04.1 LTS is getting this error: installing EmuDeck [36436:1215/124135.236209:FATAL:setuid_sandbox_host.cc(157)] The SUID sandbox helper binary was found, but is not configured correctly. Rather than run without sandboxing I'm aborting now. You need to make sure that /tmp/.mount_EmuDecSYLXPs/chrome-sandbox is owned by root and has mode 4755. bash, linha 111: 36436 Trace/breakpoint trap (imagem do núcleo gravada) ~/Applications/EmuDeck.AppImage $sandbox Something went wrong! Error at 111 NULL: ~/Applications/EmuDeck.AppImage $sandbox I just add an verification for set sandbox to "--no-sandbox" and it works! * Update install.sh Update --------- Co-authored-by: Dragoon Dorise # Conflicts: # install.sh * fix prerequisites installation # Conflicts: # install-early.sh # install-unstable.sh * Added json for Petrophobia --------- Co-authored-by: Dragoon Dorise Co-authored-by: Gabriel Cruz <48037228+gabrielpcruz@users.noreply.github.com> --- install.sh | 4 ++-- store/gbc/Petrophobia.json | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 store/gbc/Petrophobia.json diff --git a/install.sh b/install.sh index bf225a055..0aeab2903 100644 --- a/install.sh +++ b/install.sh @@ -2,7 +2,6 @@ linuxID=$(lsb_release -si) sandbox="" - if [ $linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi @@ -109,4 +108,5 @@ EMUDECK_URL="$(curl -s ${EMUDECK_GITHUB_URL} | grep -E 'browser_download_url.*Ap mkdir -p ~/Applications curl -L "${EMUDECK_URL}" -o ~/Applications/EmuDeck.AppImage 2>&1 | stdbuf -oL tr '\r' '\n' | sed -u 's/^ *\([0-9][0-9]*\).*\( [0-9].*$\)/\1\n#Download Speed\:\2/' | zenity --progress --title "Downloading EmuDeck" --width 600 --auto-close --no-cancel 2>/dev/null chmod +x ~/Applications/EmuDeck.AppImage -~/Applications/EmuDeck.AppImage $sandbox \ No newline at end of file +~/Applications/EmuDeck.AppImage $sandbox +exit \ No newline at end of file diff --git a/store/gbc/Petrophobia.json b/store/gbc/Petrophobia.json new file mode 100644 index 000000000..a0e6957ca --- /dev/null +++ b/store/gbc/Petrophobia.json @@ -0,0 +1,18 @@ +{ + "system": "gbc", + "status": "true", + "logo": "logo_gbc", + "title": "Petrophobia", + "url": "https://orbistertius.itch.io/petrophobia", + "file": "https://raw.githubusercontent.com/EmuDeck/emudeck-homebrew/main/gbc/petrophobia.zip", + "description": "A story-driven adventure game about colonialism and petroleum, loosely based on the short story 'The Mark of the Beast' by Rudyard Kipling, a passage from Thomas Pynchon's 'Against the Day', and the oil boom of the late 19th and early 20th centuries. TRIGGER WARNING: Themes of torture, racism, homophobia, and misogyny.", + "pictures": { + "screenshots": [ + "https://raw.githubusercontent.com/EmuDeck/emudeck-homebrew/main/downloaded_media/gbc/screenshots/homebrew/Petrophobia.png?raw=true" + ], + "titlescreens": [ + "https://raw.githubusercontent.com/EmuDeck/emudeck-homebrew/main/downloaded_media/gbc/titlescreens/homebrew/Petrophobia.png?raw=true" + ] + }, + "tags": ["adventure"] +} From 29ee1cf943bd7753478720f1be6f9c18883b65b3 Mon Sep 17 00:00:00 2001 From: Junian Triajianto <3881052+junian@users.noreply.github.com> Date: Mon, 30 Dec 2024 13:56:38 +0700 Subject: [PATCH 230/230] Update Lime3DS repo. (#1379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixes dolphin control issues * Revert "fixes dolphin control issues" This reverts commit 32945b947a0306418d563386cd6c0e17030cf6c4. * dolphin for real * versions * forced citra * Fix Cemu - No DSU * GreemDev * path fix * Update emuDeckXenia.sh * Update install.sh (#1369) * fixes dolphin control issues * Revert "fixes dolphin control issues" This reverts commit 32945b947a0306418d563386cd6c0e17030cf6c4. * dolphin for real * versions * forced citra * Fix Cemu - No DSU * GreemDev * path fix * Update install.sh On Ubuntu 24.04.1 LTS is getting this error: installing EmuDeck [36436:1215/124135.236209:FATAL:setuid_sandbox_host.cc(157)] The SUID sandbox helper binary was found, but is not configured correctly. Rather than run without sandboxing I'm aborting now. You need to make sure that /tmp/.mount_EmuDecSYLXPs/chrome-sandbox is owned by root and has mode 4755. bash, linha 111: 36436 Trace/breakpoint trap (imagem do núcleo gravada) ~/Applications/EmuDeck.AppImage $sandbox Something went wrong! Error at 111 NULL: ~/Applications/EmuDeck.AppImage $sandbox I just add an verification for set sandbox to "--no-sandbox" and it works! * Update install.sh Update --------- Co-authored-by: Dragoon Dorise # Conflicts: # install.sh * fix prerequisites installation # Conflicts: # install-early.sh # install-unstable.sh * Revert "GreemDev" This reverts commit be0a81ba20b02cc120707ccba4b499c35f60ee1a. * Reapply "GreemDev" This reverts commit 3b8193f0465fe3976dc26d5a1c6a29fa0a2f3dab. * new Ryu url * Update Lime3DS repo. Sadly Lime3DS was archived on Nov 17, 2024. But the binaries still available here: https://github.com/Lime3DS/lime3ds-archive/releases * Update android Lime3DS repo --------- Co-authored-by: Dragoon Dorise Co-authored-by: Gabriel Cruz <48037228+gabrielpcruz@users.noreply.github.com> --- android/functions/EmuScripts/Android_Lime3DS.sh | 4 ++-- functions/EmuScripts/emuDeckLime3DS.sh | 2 +- install.sh | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/android/functions/EmuScripts/Android_Lime3DS.sh b/android/functions/EmuScripts/Android_Lime3DS.sh index 9ef244d79..862cbc8ed 100644 --- a/android/functions/EmuScripts/Android_Lime3DS.sh +++ b/android/functions/EmuScripts/Android_Lime3DS.sh @@ -2,7 +2,7 @@ function Android_Lime3DS_install(){ setMSG "Installing Citra" - temp_url="$(getLatestReleaseURLGH "Lime3DS/Lime3DS" ".apk")" + temp_url="$(getLatestReleaseURLGH "Lime3DS/lime3ds-archive" ".apk")" temp_emu="citra" Android_ADB_dl_installAPK $temp_emu $temp_url } @@ -22,4 +22,4 @@ function Android_Lime3DS_setup(){ function Android_Lime3DS_IsInstalled(){ package="org.citra.emu" Android_ADB_appInstalled $package -} \ No newline at end of file +} diff --git a/functions/EmuScripts/emuDeckLime3DS.sh b/functions/EmuScripts/emuDeckLime3DS.sh index d02433446..1e4e716a1 100755 --- a/functions/EmuScripts/emuDeckLime3DS.sh +++ b/functions/EmuScripts/emuDeckLime3DS.sh @@ -12,7 +12,7 @@ Lime3DS_texturesPath="$HOME/.config/lime3ds-emu/load/textures" Lime3DS_install(){ echo "Begin $Lime3DS_emuName Install" local showProgress="$1" - if installEmuAI "$Lime3DS_emuName" "" "$(getReleaseURLGH "Lime3DS/Lime3DS" "tar.gz" "")" "lime3ds" "tar.gz" "emulator" "$showProgress"; then #lime3ds-gui.AppImage + if installEmuAI "$Lime3DS_emuName" "" "$(getReleaseURLGH "Lime3DS/lime3ds-archive" "tar.gz" "")" "lime3ds" "tar.gz" "emulator" "$showProgress"; then #lime3ds-gui.AppImage mkdir "$HOME/Applications/lime3ds-temp" tar -xvzf "$HOME/Applications/lime3ds.tar.gz" -C "$HOME/Applications/lime3ds-temp" --strip-components 1 if [ -f "$HOME/Applications/lime3ds-temp/lime3ds-gui.AppImage" ]; then diff --git a/install.sh b/install.sh index 0aeab2903..b439c643e 100644 --- a/install.sh +++ b/install.sh @@ -2,10 +2,12 @@ linuxID=$(lsb_release -si) sandbox="" + if [ $linuxID = "Ubuntu" ]; then sandbox="--no-sandbox" fi clear + if [ "$linuxID" == "SteamOS" ]; then echo "Installing EmuDeck" else