From 3083a1233656b31299eb9cd59274d16c17a8f99d Mon Sep 17 00:00:00 2001 From: Melissa Autumn Date: Thu, 1 Dec 2022 13:36:26 -0800 Subject: [PATCH 1/4] Add the cache-bust.py script. By default it won't bust the cache on you, see the argparse or --help for more info. --- cache-bust.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++ cacheclear.py | 38 ---------------------- 2 files changed, 88 insertions(+), 38 deletions(-) create mode 100644 cache-bust.py delete mode 100644 cacheclear.py diff --git a/cache-bust.py b/cache-bust.py new file mode 100644 index 0000000..19abe47 --- /dev/null +++ b/cache-bust.py @@ -0,0 +1,88 @@ +import sys + +sys.path.append('/home/ansibler/thunderbird-website/') + +import CloudFlare +import os + +from git import Repo +import argparse as argparse + +# these need to be in the environment +apikey = os.environ['CF_KEY'] +email = os.environ['CF_EMAIL'] +zone = os.environ['CF_ZONE_IDENTIFIER'] + +# Cloudflare has a 30-item limit on API cache purges now. +def chunks(lst, n): + for i in range(0, len(lst), n): + yield lst[i:i + n] + +def main(): + parser = argparse.ArgumentParser("Cache Buster", description="Cloudflare Cache Buster. By default it won't bust the cache unless you pass the `--bust-cache` argument.") + parser.add_argument('path', help="Path to the git directory we're diffing.") + parser.add_argument('--print-urls', action='store_true', help="Print each of the to-be-cache-busted url.") + parser.add_argument('--bust-cache', action='store_true', help="Bust the CloudFlare cache.") + args = parser.parse_args() + + path = args.path + urls = gather_urls(path) + + if args.print_urls is True: + print("Printing out urls:") + for url in urls: + print(url) + + if args.bust_cache is True: + print("Busting cache for {} urls.".format(len(urls))) + bust_cache(urls) + + print("Finished.") + + +def bust_cache(url_list): + """ Talks with cloudflare to bust the requested urls """ + cf = CloudFlare.CloudFlare(email=email, token=apikey) + + for urls in chunks(url_list, 30): + api_req = {'files': urls} + response = cf.zones.purge_cache.post(zone, data=api_req) + print(response) + +def gather_urls(repo_path): + """ Gathers list of urls from the git diff. """ + # Grab the repo from + repo = Repo(repo_path) + # Diff against our previous commit + diff = repo.head.commit.diff("HEAD~1") + + url_list = [] + + for item in diff: + # We can skip new files + if item.change_type == 'A': + continue + + # For deletes we want the path that existed prior to the deletion + if item.change_type == 'D': + path = item.a_path + else: + path = item.b_path + + # We're currently only busting thunderbird.net caches + if path is None or not path.startswith("thunderbird.net"): + continue + + # This will form https://www.thunderbird.net/ etc... + url = "https://www.{}".format(path) + + # If it's an index page, remove the index.html + if "index.html" in url: + url = url.replace('index.html', '') + + url_list.append(url) + + return url_list + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/cacheclear.py b/cacheclear.py deleted file mode 100644 index b2ce079..0000000 --- a/cacheclear.py +++ /dev/null @@ -1,38 +0,0 @@ -import sys -sys.path.append('/home/ansibler/thunderbird-website/') - -import CloudFlare -import os -import settings - -# these need to be in the environment -apikey = os.environ['CF_KEY'] -email = os.environ['CF_EMAIL'] -zone = os.environ['CF_ZONE_IDENTIFIER'] - -# Cloudflare has a 30-item limit on API cache purges now. -def chunks(lst, n): - for i in xrange(0, len(lst), n): - yield lst[i:i + n] - -# Build URL list -tb = 'https://www.thunderbird.net/' -urls = [] - -for lang in settings.PROD_LANGUAGES: - urls.append(tb+lang+'/') - -urls.append('https://www.thunderbird.net/en-US/thunderbird/releases/') -urls.append('https://www.thunderbird.net/en-US/thunderbird/all/') -urls.append('https://www.thunderbird.net/media/js/common-bundle.js') - -cf = CloudFlare.CloudFlare(email=email, token=apikey) - -lists = chunks(urls, 30) # Split into 30-item requests. -for url_list in lists: - # Put it in the format the API expects. - api_req = {'files': url_list} - - a = cf.zones.purge_cache.post(zone, data=api_req) - print a -print "Completed." From 5272729f9312c3e36e9cf806e973a391ee8cb3f9 Mon Sep 17 00:00:00 2001 From: Melissa Autumn Date: Thu, 1 Dec 2022 13:42:16 -0800 Subject: [PATCH 2/4] Add a bust cache website build option --- plays/website-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plays/website-build.yml b/plays/website-build.yml index f984f1d..0c0a20f 100644 --- a/plays/website-build.yml +++ b/plays/website-build.yml @@ -108,6 +108,11 @@ become: yes become_user: ansibler + - name: bust website cache + command: "python cache-bust.py {{ tbsitebuild }} --bust-cache" + become: yes + become_user: ansibler + - name: clean up temporary site files file: path: "{{ item }}" From 3856a05c5ea5a74305a48b2983ca2bea0ba10338 Mon Sep 17 00:00:00 2001 From: Melissa Autumn Date: Thu, 1 Dec 2022 14:02:09 -0800 Subject: [PATCH 3/4] Gracefully exit if we get hit by a cloudflare api error. (i.e. rate limits) --- cache-bust.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cache-bust.py b/cache-bust.py index 19abe47..fc785af 100644 --- a/cache-bust.py +++ b/cache-bust.py @@ -2,7 +2,7 @@ sys.path.append('/home/ansibler/thunderbird-website/') -import CloudFlare +from CloudFlare import CloudFlare import os from git import Repo @@ -42,12 +42,19 @@ def main(): def bust_cache(url_list): """ Talks with cloudflare to bust the requested urls """ - cf = CloudFlare.CloudFlare(email=email, token=apikey) + cf = CloudFlare(email=email, token=apikey) for urls in chunks(url_list, 30): api_req = {'files': urls} - response = cf.zones.purge_cache.post(zone, data=api_req) - print(response) + # See: https://developers.cloudflare.com/api/operations/zone-purge-files-by-cache-tags,-host,-or-prefix + + try: + response = cf.zones.purge_cache.post(zone, data=api_req) + print("Cloudflare Response: {}".format(response)) + except CloudFlare.exceptions.CloudFlareAPIError as err: + # If there's any errors with busting the cache, then just stop doing it. Don't want to prevent the deployment. + print("Error busting cache: {}".format(err)) + break def gather_urls(repo_path): """ Gathers list of urls from the git diff. """ From 413dcbfe0211277b5b268104fc8c76862780cdc3 Mon Sep 17 00:00:00 2001 From: Melissa Autumn Date: Thu, 1 Dec 2022 14:06:47 -0800 Subject: [PATCH 4/4] Small clean up --- cache-bust.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cache-bust.py b/cache-bust.py index fc785af..4bf28d6 100644 --- a/cache-bust.py +++ b/cache-bust.py @@ -46,9 +46,8 @@ def bust_cache(url_list): for urls in chunks(url_list, 30): api_req = {'files': urls} - # See: https://developers.cloudflare.com/api/operations/zone-purge-files-by-cache-tags,-host,-or-prefix - try: + # See: https://developers.cloudflare.com/api/operations/zone-purge-files-by-cache-tags,-host,-or-prefix response = cf.zones.purge_cache.post(zone, data=api_req) print("Cloudflare Response: {}".format(response)) except CloudFlare.exceptions.CloudFlareAPIError as err: