Merge pull request #86 from aarongustafson/full-throttle

Full throttle

lib/jekyll/generators/gather_webmentions.rb

@@ -35,11 +35,15 @@ def generate(site)
 
       @cached_webmentions = Jekyll::WebmentionIO.read_cached_webmentions "incoming"
 
+      @lookups = Jekyll::WebmentionIO.read_lookup_dates
+
       posts = Jekyll::WebmentionIO.gather_documents(@site)
       posts.each do |post|
         check_for_webmentions(post)
       end
 
+      Jekyll::WebmentionIO.cache_lookup_dates @lookups
+
       Jekyll::WebmentionIO.cache_webmentions "incoming", @cached_webmentions
     end # generate
 
@@ -48,12 +52,18 @@ def generate(site)
     def check_for_webmentions(post)
       Jekyll::WebmentionIO.log "info", "Checking for webmentions of #{post.url}."
 
-      # get the last webmention
       last_webmention = @cached_webmentions.dig(post.url, @cached_webmentions.dig(post.url)&.keys&.last)
 
+      # get the last webmention
+      last_lookup = if @lookups[post.url]
+                      @lookups[post.url]
+                    else
+                      Date.parse last_webmention.dig("raw", "verified_date")
+                    end
+
       # should we throttle?
       if post.respond_to? "date" # Some docs have no date
-        if last_webmention && Jekyll::WebmentionIO.post_should_be_throttled?(post, post.date, last_webmention.dig("raw", "verified_date"))
+        if last_lookup && Jekyll::WebmentionIO.post_should_be_throttled?(post, post.date, last_lookup)
           Jekyll::WebmentionIO.log "info", "Throttling this post."
           return
         end
@@ -74,6 +84,7 @@ def check_for_webmentions(post)
         Jekyll::WebmentionIO.log "info", "No webmentions found."
       end
 
+      @lookups[post.url] = Date.today
       cache_new_webmentions(post.url, response)
     end
 

lib/jekyll/webmention_io.rb

@@ -49,6 +49,7 @@ def self.bootstrap
         "incoming" => "#{@cache_folder}/#{@file_prefix}received.yml",
         "outgoing" => "#{@cache_folder}/#{@file_prefix}outgoing.yml",
         "bad_uris" => "#{@cache_folder}/#{@file_prefix}bad_uris.yml",
+        "lookups"  => "#{@cache_folder}/#{@file_prefix}lookups.yml"
       }
       @cache_files.each do |_key, file|
         unless File.exist?(file)
@@ -160,14 +161,27 @@ def self.get_response(api_params)
       end
     end
 
+    def self.read_lookup_dates()
+      cache_file = get_cache_file_path "lookups"
+      lookups = open(cache_file) { |f| YAML.load(f) }
+      lookups
+    end
+
+    def self.cache_lookup_dates(lookups)
+      cache_file = get_cache_file_path "lookups"
+      File.open(cache_file, "w") { |f| YAML.dump(lookups, f) }
+
+      Jekyll::WebmentionIO.log "msg", "Lookups have been cached."
+    end
+
     # allowed throttles: last_week, last_month, last_year, older
     # allowed values:  daily, weekly, monthly, yearly, every X days|weeks|months|years
-    def self.post_should_be_throttled?(post, item_date, last_webmention_date)
+    def self.post_should_be_throttled?(post, item_date, last_lookup)
       throttles = @config.dig("throttle_lookups")
-      if throttles && item_date && last_webmention_date
+      if throttles && item_date && last_lookup
         age = get_timeframe_from_date(item_date)
         throttle = throttles.dig(age)
-        if throttle && Date.parse(last_webmention_date) >= get_date_from_string(throttle)
+        if throttle && last_lookup >= get_date_from_string(throttle)
           log "info", "Throttling #{post.data["title"]} (Only checking it #{throttle})"
           return true
         end

lib/jekyll/webmention_io/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module WebmentionIO
-    VERSION = "2.9.3".freeze
+    VERSION = "2.9.4".freeze
   end
 end