Add Memcached adapter for GitCache

Author: root

Diff for: Gemfile

@@ -9,6 +9,9 @@ gem 'gitlab-grack', git: 'https://github.com/n-rodriguez/grack.git', require: 'g
 # HAML views
 gem 'haml-rails'
 
+# Memcached client for GitCache
+gem 'dalli'
+
 # Syntaxic coloration
 gem 'github-markup'
 gem 'redcarpet', '~> 2.3.0'

Diff for: app/helpers/gitolite_plugin_settings_helper.rb

@@ -79,6 +79,14 @@ def git_cache_options
   end
 
 
+  def git_cache_adapters
+    [
+      ['Database', 'database'],
+      ['Memcached', 'memcached']
+    ]
+  end
+
+
   def log_level_options
     [
       [l(:label_debug), 'debug'],

Diff for: app/services/gitolite_accessor.rb

@@ -76,7 +76,7 @@ def purge_trash_bin(repositories)
 
     def flush_git_cache
       logger.info('Flush Git Cache !')
-      ActiveRecord::Base.connection.execute('TRUNCATE git_caches')
+      RedmineGitHosting::Cache.flush_cache!
     end
 
 

Diff for: app/views/settings/_gitolite_config_cache.html.haml

@@ -2,6 +2,7 @@
 - gitolite_cache_max_time     = RedmineGitHosting::Config.get_setting(:gitolite_cache_max_time)
 - gitolite_cache_max_size     = RedmineGitHosting::Config.get_setting(:gitolite_cache_max_size)
 - gitolite_cache_max_elements = RedmineGitHosting::Config.get_setting(:gitolite_cache_max_elements)
+- gitolite_cache_adapter      = RedmineGitHosting::Config.get_setting(:gitolite_cache_adapter)
 
 %h3= l(:label_gitolite_cache_config)
 
@@ -20,3 +21,8 @@
   %label= l(:label_gitolite_cache_max_elements)
   = text_field_tag 'settings[gitolite_cache_max_elements]', gitolite_cache_max_elements, size: 20
   %br
+
+%p
+  %label= l(:label_gitolite_cache_adapter)
+  = select_tag 'settings[gitolite_cache_adapter]', options_for_select(git_cache_adapters, gitolite_cache_adapter)
+  %br

Diff for: config/locales/settings/en.yml

@@ -100,6 +100,7 @@ en:
   label_gitolite_cache_max_time: Max cache time
   label_gitolite_cache_max_size: Max cache element size
   label_gitolite_cache_max_elements: Max cache elements
+  label_gitolite_cache_adapter: Cache Adapter
 
   # Gitolite Notifications
   label_tab_notify: Notifications

Diff for: config/locales/settings/fr.yml

@@ -100,6 +100,7 @@ fr:
   label_gitolite_cache_max_time: Temps maximal du cache
   label_gitolite_cache_max_size: Taille maximale d'un élément du cache
   label_gitolite_cache_max_elements: Nombre maximal d'éléments dans le cache
+  label_gitolite_cache_adapter: Cache Adapter
 
   # Gitolite Notifications
   label_tab_notify: Notifications

Diff for: init.rb

@@ -50,6 +50,7 @@
       :gitolite_cache_max_time          => '86400',
       :gitolite_cache_max_size          => '16',
       :gitolite_cache_max_elements      => '2000',
+      :gitolite_cache_adapter           => 'database',
 
       # Gitolite Access Config
       :ssh_server_domain                => 'localhost',

Diff for: lib/redmine_git_hosting/cache.rb

@@ -3,16 +3,6 @@ module Cache
 
     class << self
 
-      def max_cache_time
-        RedmineGitHosting::Config.gitolite_cache_max_time
-      end
-
-
-      def max_cache_elements
-        RedmineGitHosting::Config.gitolite_cache_max_elements
-      end
-
-
       # Used in ShellRedirector but define here to keep a clean interface.
       def max_cache_size
         RedmineGitHosting::Config.gitolite_cache_max_size
@@ -21,37 +11,26 @@ def max_cache_size
 
       def set_cache(repo_id, out_value, primary_key, secondary_key = nil)
         command = compose_key(primary_key, secondary_key)
-        adapter.apply_cache_limit(max_cache_elements) if adapter.set_cache(command, out_value, repo_id)
+        adapter.apply_cache_limit if adapter.set_cache(command, out_value, repo_id)
       end
 
 
       def get_cache(primary_key, secondary_key = nil)
         command = compose_key(primary_key, secondary_key)
         cached  = adapter.get_cache(command)
+        # Return result as a string stream
+        cached.nil? ? nil : StringIO.new(cached)
+      end
 
-        if cached
-          if valid_cache_entry?(cached.created_at)
-            # Update updated_at flag
-            cached.touch unless cached.command_output.nil?
-            out = cached.command_output
-          else
-            cached.destroy
-            out = nil
-          end
-        else
-          out = nil
-        end
 
-        # Return result as a string stream
-        out.nil? ? nil : StringIO.new(out)
+      def flush_cache!
+        adapter.flush_cache!
       end
 
 
       # After resetting cache timing parameters -- delete entries that no-longer match
       def clear_obsolete_cache_entries
-        return if max_cache_time < 0  # No expiration needed
-        limit = Time.now - max_cache_time
-        adapter.clear_obsolete_cache_entries(limit)
+        adapter.clear_obsolete_cache_entries
       end
 
 
@@ -62,20 +41,13 @@ def clear_cache_for_repository(repo_id)
 
 
       def adapter
-        @adapter ||= Cache::Adapter.factory
+        Cache::Adapter.factory
       end
 
 
       private
 
 
-        def valid_cache_entry?(cached_entry_date)
-          current_time = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
-          expired = (current_time.to_i - cached_entry_date.to_i > max_cache_time)
-          (!expired || max_cache_time < 0) ? true : false
-        end
-
-
         def compose_key(key1, key2)
           if key2 && !key2.blank?
             key1 + "\n" + key2

Diff for: lib/redmine_git_hosting/cache/abstract_cache.rb

@@ -1,6 +1,18 @@
 module RedmineGitHosting::Cache
   class AbstractCache
 
+    attr_reader :max_cache_size
+    attr_reader :max_cache_time
+    attr_reader :max_cache_elements
+
+
+    def initialize
+      @max_cache_size     = RedmineGitHosting::Config.gitolite_cache_max_size
+      @max_cache_time     = RedmineGitHosting::Config.gitolite_cache_max_time
+      @max_cache_elements = RedmineGitHosting::Config.gitolite_cache_max_elements
+    end
+
+
     def set_cache(command, output, repo_id)
       raise NotImplementedError
     end
@@ -11,7 +23,12 @@ def get_cache(command)
     end
 
 
-    def clear_obsolete_cache_entries(limit)
+    def flush_cache!
+      raise NotImplementedError
+    end
+
+
+    def clear_obsolete_cache_entries
       raise NotImplementedError
     end
 
@@ -21,7 +38,7 @@ def clear_cache_for_repository(repo_id)
     end
 
 
-    def apply_cache_limit(max_cache_elements)
+    def apply_cache_limit
       raise NotImplementedError
     end
 
@@ -33,5 +50,19 @@ def logger
         RedmineGitHosting.logger
       end
 
+
+      def time_limit
+        return if max_cache_time < 0  # No expiration needed
+        current_time = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
+        limit = current_time - max_cache_time
+      end
+
+
+      def valid_cache_entry?(cached_entry_date)
+        current_time = ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
+        expired = (current_time.to_i - cached_entry_date.to_i > max_cache_time)
+        (!expired || max_cache_time < 0) ? true : false
+      end
+
   end
 end

Diff for: lib/redmine_git_hosting/cache/adapter.rb

@@ -4,7 +4,12 @@ class Adapter
     class << self
 
       def factory
-        Database.new
+        case RedmineGitHosting::Config.gitolite_cache_adapter
+        when 'database'
+          Database.new
+        when 'memcached'
+          Memcached.new
+        end
       end
 
     end

Diff for: lib/redmine_git_hosting/cache/database.rb

@@ -2,7 +2,7 @@ module RedmineGitHosting::Cache
   class Database < AbstractCache
 
     def set_cache(command, output, repo_id)
-      logger.debug("Inserting cache entry for repository '#{repo_id}'")
+      logger.debug("DB Adapter : inserting cache entry for repository '#{repo_id}'")
       begin
         GitCache.create(
           command:         command,
@@ -11,30 +11,48 @@ def set_cache(command, output, repo_id)
         )
         true
       rescue => e
-        logger.error("Could not insert in cache, this is the error : '#{e.message}'")
+        logger.error("DB Adapter : could not insert in cache, this is the error : '#{e.message}'")
         false
       end
     end
 
 
     def get_cache(command)
-      GitCache.find_by_command(command)
+      cached = GitCache.find_by_command(command)
+      if cached
+        if valid_cache_entry?(cached.created_at)
+          # Update updated_at flag
+          cached.touch unless cached.command_output.nil?
+          out = cached.command_output
+        else
+          cached.destroy
+          out = nil
+        end
+      else
+        out = nil
+      end
+      out
+    end
+
+
+    def flush_cache!
+      ActiveRecord::Base.connection.execute('TRUNCATE git_caches')
     end
 
 
-    def clear_obsolete_cache_entries(limit)
-      deleted = GitCache.delete_all(['created_at < ?', limit])
-      logger.info("Removed '#{deleted}' expired cache entries among all repositories")
+    def clear_obsolete_cache_entries
+      deleted = GitCache.delete_all(['created_at < ?', time_limit])
+      logger.info("DB Adapter : removed '#{deleted}' expired cache entries among all repositories")
     end
 
 
     def clear_cache_for_repository(repo_id)
       deleted = GitCache.delete_all(['repo_identifier = ?', repo_id])
-      logger.info("Removed '#{deleted}' expired cache entries for repository '#{repo_id}'")
+      logger.info("DB Adapter : removed '#{deleted}' expired cache entries for repository '#{repo_id}'")
     end
 
 
-    def apply_cache_limit(max_cache_elements)
+    def apply_cache_limit
       GitCache.find(:last, order: 'created_at DESC').destroy if max_cache_elements >= 0 && GitCache.count > max_cache_elements
     end
 

Diff for: lib/redmine_git_hosting/cache/memcached.rb

@@ -0,0 +1,95 @@
+require 'dalli'
+require 'digest/sha1'
+
+module RedmineGitHosting::Cache
+  class Memcached < AbstractCache
+
+    def set_cache(command, output, repo_id)
+      logger.debug("Memcached Adapter : inserting cache entry for repository '#{repo_id}'")
+
+      # Create a SHA256 of the Git command as key id
+      hashed_command = hash_key(command)
+
+      begin
+        create_or_update_repo_references(repo_id, hashed_command)
+        client.set(hashed_command, output)
+        true
+      rescue => e
+        logger.error("Memcached Adapter : could not insert in cache, this is the error : '#{e.message}'")
+        false
+      end
+    end
+
+
+    def get_cache(command)
+      client.get(hash_key(command))
+    end
+
+
+    def flush_cache!
+      client.flush
+    end
+
+
+    # Return true, this is done automatically by Memcached with the
+    # *max_cache_time* params (see below)
+    #
+    def clear_obsolete_cache_entries
+      true
+    end
+
+
+    def clear_cache_for_repository(repo_id)
+      # Create a SHA256 of the repo_id as key id
+      hashed_repo_id = hash_key(repo_id)
+      # Find repository references in Memcached
+      repo_references = client.get(hashed_repo_id)
+      return true if repo_references.nil?
+      # Delete reference keys
+      repo_references = repo_references.split(',').select { |r| !r.empty? }
+      repo_references.map { |key| client.delete(key) }
+      logger.info("Memcached Adapter : removed '#{repo_references.size}' expired cache entries for repository '#{repo_id}'")
+      # Reset references count
+      client.set(hashed_repo_id, '', max_cache_time, raw: true)
+    end
+
+
+    # Return true. If cache is full, Memcached drop the oldest objects to add new ones.
+    #
+    def apply_cache_limit
+      true
+    end
+
+
+    private
+
+
+      def create_or_update_repo_references(repo_id, reference)
+        # Create a SHA256 of the repo_id as key id
+        hashed_repo_id = hash_key(repo_id)
+        # Find it in Memcached
+        repo_references = client.get(hashed_repo_id)
+        if repo_references.nil?
+          client.set(hashed_repo_id, reference, max_cache_time, raw: true)
+        else
+          client.append(hashed_repo_id, ',' + reference)
+        end
+      end
+
+
+      def hash_key(key)
+        Digest::SHA256.hexdigest(key)
+      end
+
+
+      def client
+        @client ||= Dalli::Client.new('localhost:11211', memcached_options)
+      end
+
+
+      def memcached_options
+        { namespace: 'redmine_git_hosting', compress: true, expires_in: max_cache_time, value_max_bytes: max_cache_size }
+      end
+
+  end
+end