From 24118b9ddcea8a2bd56f8b77bf40cf4b2490c683 Mon Sep 17 00:00:00 2001 From: A Thousand Ships <96648715+AThousandShips@users.noreply.github.com> Date: Sat, 9 Nov 2024 11:56:34 +0100 Subject: [PATCH 1/4] [Buildsystem] Improve cache handling --- SConstruct | 3 +- methods.py | 99 ++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 64 insertions(+), 38 deletions(-) diff --git a/SConstruct b/SConstruct index d0ffcde3e35b..48632cbaa8e3 100644 --- a/SConstruct +++ b/SConstruct @@ -796,12 +796,13 @@ elif selected_platform != "": # The following only makes sense when the 'env' is defined, and assumes it is. if "env" in locals(): - # FIXME: This method mixes both cosmetic progress stuff and cache handling... methods.show_progress(env) # TODO: replace this with `env.Dump(format="json")` # once we start requiring SCons 4.0 as min version. methods.dump(env) + methods.clean_cache(env) + def print_elapsed_time(): elapsed_time_sec = round(time.time() - time_at_start, 3) diff --git a/methods.py b/methods.py index 07469f60be1d..d125c119e545 100644 --- a/methods.py +++ b/methods.py @@ -1126,21 +1126,19 @@ def show_progress(env): "fname": str(env.Dir("#")) + "/.scons_node_count", } - import time, math + import math class cache_progress: - # The default is 1 GB cache and 12 hours half life - def __init__(self, path=None, limit=1073741824, half_life=43200): + # The default is 1 GB cache + def __init__(self, path=None, limit=pow(1024, 3)): self.path = path self.limit = limit - self.exponent_scale = math.log(2) / half_life if env["verbose"] and path != None: screen.write( "Current cache limit is {} (used: {})\n".format( self.convert_size(limit), self.convert_size(self.get_size(path)) ) ) - self.delete(self.file_list()) def __call__(self, node, *args, **kw): if show_progress: @@ -1158,12 +1156,65 @@ def __call__(self, node, *args, **kw): screen.write("\r[Initial build] ") screen.flush() + def convert_size(self, size_bytes): + if size_bytes == 0: + return "0 bytes" + size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = int(math.floor(math.log(size_bytes, 1024))) + p = math.pow(1024, i) + s = round(size_bytes / p, 2) + return "%s %s" % (int(s) if i == 0 else s, size_name[i]) + + def get_size(self, start_path="."): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(start_path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + + def progress_finish(target, source, env): + try: + with open(node_count_data["fname"], "w") as f: + f.write("%d\n" % node_count_data["count"]) + except Exception: + pass + + try: + with open(node_count_data["fname"]) as f: + node_count_data["max"] = int(f.readline()) + except Exception: + pass + + cache_directory = os.environ.get("SCONS_CACHE") + # Simple cache pruning, attached to SCons' progress callback. Trim the + # cache directory to a size not larger than cache_limit. + cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 + progressor = cache_progress(cache_directory, cache_limit) + Progress(progressor, interval=node_count_data["interval"]) + + progress_finish_command = Command("progress_finish", [], progress_finish) + AlwaysBuild(progress_finish_command) + + +def clean_cache(env): + import atexit + import time + + class cache_clean: + def __init__(self, path=None, limit=pow(1024, 3)): + self.path = path + self.limit = limit + + def clean(self): + self.delete(self.file_list()) + def delete(self, files): if len(files) == 0: return if env["verbose"]: # Utter something - screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file")) + print("Purging %d %s from cache..." % (len(files), "files" if len(files) > 1 else "file")) [os.remove(f) for f in files] def file_list(self): @@ -1197,46 +1248,20 @@ def file_list(self): else: return [x[0] for x in file_stat[mark:]] - def convert_size(self, size_bytes): - if size_bytes == 0: - return "0 bytes" - size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") - i = int(math.floor(math.log(size_bytes, 1024))) - p = math.pow(1024, i) - s = round(size_bytes / p, 2) - return "%s %s" % (int(s) if i == 0 else s, size_name[i]) - - def get_size(self, start_path="."): - total_size = 0 - for dirpath, dirnames, filenames in os.walk(start_path): - for f in filenames: - fp = os.path.join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size - - def progress_finish(target, source, env): + def cache_finally(): + nonlocal cleaner try: - with open(node_count_data["fname"], "w") as f: - f.write("%d\n" % node_count_data["count"]) - progressor.delete(progressor.file_list()) + cleaner.clean() except Exception: pass - try: - with open(node_count_data["fname"]) as f: - node_count_data["max"] = int(f.readline()) - except Exception: - pass - cache_directory = os.environ.get("SCONS_CACHE") # Simple cache pruning, attached to SCons' progress callback. Trim the # cache directory to a size not larger than cache_limit. cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 - progressor = cache_progress(cache_directory, cache_limit) - Progress(progressor, interval=node_count_data["interval"]) + cleaner = cache_clean(cache_directory, cache_limit) - progress_finish_command = Command("progress_finish", [], progress_finish) - AlwaysBuild(progress_finish_command) + atexit.register(cache_finally) def dump(env): From feeb0721abbd64d2e46423d02630fed31c42aa5d Mon Sep 17 00:00:00 2001 From: A Thousand Ships <96648715+AThousandShips@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:54:03 +0100 Subject: [PATCH 2/4] [JavaScript] Don't cache emsdk Due to how caches are accessed this cache is almost useless, it only matters if it is from the same branch or a base branch, and is identical between branches, so caching it just clutters the build cache --- .github/workflows/javascript_builds.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/javascript_builds.yml b/.github/workflows/javascript_builds.yml index 8739a65307ee..895869063b67 100644 --- a/.github/workflows/javascript_builds.yml +++ b/.github/workflows/javascript_builds.yml @@ -8,7 +8,6 @@ env: GODOT_BASE_BRANCH: 3.x SCONSFLAGS: verbose=yes warnings=all werror=yes debug_symbols=no EM_VERSION: 3.1.39 - EM_CACHE_FOLDER: "emsdk-cache" concurrency: group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}-javascript @@ -26,8 +25,7 @@ jobs: uses: mymindstorm/setup-emsdk@v14 with: version: ${{env.EM_VERSION}} - actions-cache-folder: ${{env.EM_CACHE_FOLDER}} - cache-key: emsdk-${{ matrix.cache-name }}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}} + no-cache: true - name: Verify Emscripten setup run: | From 0e674d74f2216bcfadec25262cf2484b782f9bc8 Mon Sep 17 00:00:00 2001 From: arkology <43543909+arkology@users.noreply.github.com> Date: Mon, 11 Nov 2024 06:34:03 +0000 Subject: [PATCH 3/4] [3.x] Document Timer autostart in tool scripts --- doc/classes/Timer.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/classes/Timer.xml b/doc/classes/Timer.xml index 10b6e2e22ff4..f4f9e3c0d332 100644 --- a/doc/classes/Timer.xml +++ b/doc/classes/Timer.xml @@ -37,6 +37,7 @@ If [code]true[/code], the timer will automatically start when entering the scene tree. [b]Note:[/b] This property is automatically set to [code]false[/code] after the timer enters the scene tree and starts. + [b]Note:[/b] This property does nothing when the timer is running in the editor. If [code]true[/code], the timer will stop when reaching 0. If [code]false[/code], it will restart. From 4c930bb3387ec9da6c3502caa09260893b156a34 Mon Sep 17 00:00:00 2001 From: lawnjelly Date: Fri, 22 Nov 2024 12:13:35 +0000 Subject: [PATCH 4/4] Ameliorate performance regression due to directional shadow `fade_start` --- drivers/gles3/rasterizer_scene_gles3.cpp | 5 ++++- drivers/gles3/rasterizer_scene_gles3.h | 3 +-- drivers/gles3/shaders/scene.glsl | 8 +++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/gles3/rasterizer_scene_gles3.cpp b/drivers/gles3/rasterizer_scene_gles3.cpp index 029625d01ebe..63517c53fd9f 100644 --- a/drivers/gles3/rasterizer_scene_gles3.cpp +++ b/drivers/gles3/rasterizer_scene_gles3.cpp @@ -2799,7 +2799,10 @@ void RasterizerSceneGLES3::_setup_directional_light(int p_index, const Transform const float fade_start = li->light_ptr->param[VS::LIGHT_PARAM_SHADOW_FADE_START]; // Using 1.0 would break `smoothstep()` in the shader. ubo_data.fade_from = -ubo_data.shadow_split_offsets[shadow_count - 1] * MIN(fade_start, 0.999); - ubo_data.fade_to = -ubo_data.shadow_split_offsets[shadow_count - 1]; + + // To prevent the need for a fade to, store the fade to in the final split offset. + // It will either be the same as before, or the maximum split offset. + ubo_data.shadow_split_offsets[3] = ubo_data.shadow_split_offsets[shadow_count - 1]; } glBindBuffer(GL_UNIFORM_BUFFER, state.directional_ubo); diff --git a/drivers/gles3/rasterizer_scene_gles3.h b/drivers/gles3/rasterizer_scene_gles3.h index 52d4cc544158..ab0d80e9557e 100644 --- a/drivers/gles3/rasterizer_scene_gles3.h +++ b/drivers/gles3/rasterizer_scene_gles3.h @@ -592,8 +592,7 @@ class RasterizerSceneGLES3 : public RasterizerScene { float shadow_split_offsets[4]; float fade_from; - float fade_to; - float pad[2]; + float pad[3]; }; struct LightInstance : public RID_Data { diff --git a/drivers/gles3/shaders/scene.glsl b/drivers/gles3/shaders/scene.glsl index 266db3317ddc..03515c728f3a 100644 --- a/drivers/gles3/shaders/scene.glsl +++ b/drivers/gles3/shaders/scene.glsl @@ -148,8 +148,7 @@ layout(std140) uniform DirectionalLightData { //ubo:3 mediump vec4 shadow_split_offsets; mediump float fade_from; - mediump float fade_to; - mediump vec2 pad; + mediump vec3 pad; }; #endif //ubershader-skip @@ -848,8 +847,7 @@ layout(std140) uniform DirectionalLightData { mediump vec4 shadow_split_offsets; mediump float fade_from; - mediump float fade_to; - mediump vec2 pad; + mediump vec3 pad; }; uniform highp sampler2DShadow directional_shadow; // texunit:-5 @@ -2292,7 +2290,7 @@ FRAGMENT_SHADER_CODE shadow = min(shadow, contact_shadow); } #endif //ubershader-runtime - float pssm_fade = smoothstep(fade_from, fade_to, vertex.z); + float pssm_fade = smoothstep(fade_from, -shadow_split_offsets.w, vertex.z); light_attenuation = mix(mix(shadow_color_contact.rgb, vec3(1.0), shadow), vec3(1.0), pssm_fade); }