First commit of redis-rdb.

Author: nrk

.gitignore

@@ -0,0 +1 @@
+*.gem

LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Daniele Alessandri
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,66 @@
+# redis-rdb #
+
+This library provides a set of modules and classes that make it easy to handle binary
+database dumps generated by [Redis](http://redis.io) (.rdb files) in Ruby.
+
+Currently redis-rdb allows developers to read .rdb files in a streamable flashion
+with `RDB::Reader` by providing a set of callbacks. Database objects can optionally
+be filtered by database / key / type using custom filters.
+
+```ruby
+require 'rdb'
+
+class MyCallbacks
+  include RDB::ReaderCallbacks
+
+  def set(key, value, state)
+    puts "SET \"#{key}\" \"#{value}\""
+  end
+end
+
+class MyFilter
+  include RDB::ObjectFilter
+
+  KEY_SELECTOR = Regexp.compile(/user:\d+/)
+
+  def accept_object?(state)
+    state.database == 15 && KEY_SELECTOR.match(state.key)
+  end
+end
+
+RDB::Reader.read_file('dump.rdb', callbacks: MyCallbacks.new, filter: MyFilter.new)
+```
+
+For more details about callbacks and filters you can take a look at the source code in
+[lib/rdb/callbacks.rb](https://github.com/nrk/redis-rdb/lib/rdb/callbacks.rb) and
+[lib/rdb/filters.rb](https://github.com/nrk/redis-rdb/lib/rdb/filters.rb).
+
+## Additional notes and credits ##
+
+Right now there is still no documentation about the binary format of .rdb files so we used
+the code in [rdb.c](https://github.com/antirez/redis/blob/unstable/src/rdb.c) as the reference
+implementation. Credit goes also to [sripathikrishnan](https://github.com/sripathikrishnan),
+his work on the [redis-rdb-tools](https://github.com/sripathikrishnan/redis-rdb-tools) Python
+library was quite an inspiration for the final design of `RDB::Reader` and we also reused the
+.rdb files shipped with his project for testing.
+
+## Dependencies ##
+- Ruby >= 1.9.0
+
+## Links ##
+
+### Project ###
+- [Source code](https://github.com/nrk/redis-rdb/)
+- [Issue tracker](https://github.com/nrk/redis-rdb/issues)
+
+### Related ###
+- [Redis](http://redis.io/)
+- [redis-rdb-tools](https://github.com/sripathikrishnan/redis-rdb-tools) (Python)
+
+## Author ##
+
+- [Daniele Alessandri](mailto:[email protected]) ([twitter](http://twitter.com/JoL1hAHN))
+
+## License ##
+
+The code for redis-rdb is distributed under the terms of the MIT license (see LICENSE).

Rakefile

@@ -0,0 +1,7 @@
+require 'cutest'
+
+task :test do
+  Cutest.run(Dir['test/test_*.rb'])
+end
+
+task :default => :test

examples/read_rdb.rb

@@ -0,0 +1,5 @@
+$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
+
+require 'rdb'
+
+RDB::Reader.read_file('test/rdb/multiple_databases.rdb', callbacks: RDB::DebugCallbacks.new)

lib/rdb.rb

@@ -0,0 +1,9 @@
+require 'stringio'
+require 'rdb/version'
+require 'rdb/constants'
+require 'rdb/errors'
+require 'rdb/lzf'
+require 'rdb/filters'
+require 'rdb/callbacks'
+require 'rdb/reader-state'
+require 'rdb/reader'

lib/rdb/callbacks.rb

@@ -0,0 +1,125 @@
+module RDB
+  module ReaderCallbacks
+    def start_rdb(rdb_version); end
+
+    def end_rdb(); end
+
+    def start_database(database); end
+
+    def end_database(database); end
+
+    def pexpireat(key, expiration, state); end
+
+    def set(key, value, state); end
+
+    def start_list(key, length, state); end
+
+    def rpush(key, value, state); end
+
+    def end_list(key, state); end
+
+    def start_set(key, length, state); end
+
+    def sadd(key, value, state); end
+
+    def end_set(key, state); end
+
+    def start_sortedset(key, length, state); end
+
+    def zadd(key, score, value, state); end
+
+    def end_sortedset(key, state); end
+
+    def start_hash(key, length, state); end
+
+    def hset(key, field, value, state); end
+
+    def end_hash(key, state); end
+
+    def skip_object(key, state); end
+  end
+
+  class EmptyCallbacks
+    include ReaderCallbacks
+  end
+
+  class DebugCallbacks
+    include ReaderCallbacks
+
+    def start_rdb(version)
+      puts "Start RDB file - version #{version}"
+    end
+
+    def end_rdb()
+      puts "Close RDB file"
+    end
+
+    def start_database(database)
+      puts "Open database #{database}."
+    end
+
+    def end_database(database)
+      puts "Close database #{database}."
+    end
+
+    def pexpireat(key, expiration, state)
+      puts "PEXPIREAT \"#{key}\" \"#{expiration}\""
+    end
+
+    def set(key, value, state)
+      puts "SET \"#{key}\" \"#{value}\""
+    end
+
+    def start_list(key, length, state)
+      puts "Start list \"#{key}\" of #{length} items."
+    end
+
+    def rpush(key, value, state)
+      puts "RPUSH \"#{key}\" \"#{value}\""
+    end
+
+    def end_list(key, state)
+      puts "End list \"#{key}\"."
+    end
+
+    def start_set(key, length, state)
+      puts "Start set \"#{key}\" of #{length} members."
+    end
+
+    def sadd(key, value, state)
+      puts "SADD \"#{key}\" \"#{value}\""
+    end
+
+    def end_set(key, state)
+      puts "End set \"#{key}\"."
+    end
+
+    def start_sortedset(key, length, state)
+      puts "Start sortedset \"#{key}\" of #{length} members."
+    end
+
+    def zadd(key, score, value, state)
+      puts "ZADD \"#{key}\" \"#{score}\" \"#{value}\""
+    end
+
+    def end_sortedset(key, state)
+      puts "End sortedset \"#{key}\"."
+    end
+
+    def start_hash(key, length, state)
+      puts "Start hash \"#{key}\" of #{length} members."
+    end
+
+    def hset(key, field, value, state)
+      puts "HSET \"#{key}\" \"#{field}\" \"#{value}\""
+    end
+
+    def end_hash(key, state)
+      puts "End hash \"#{key}\"."
+    end
+
+    def skip_object(key, state)
+      puts "Skipping object for key #{key} of type #{state.mnemonic_type}"
+    end
+  end
+end

lib/rdb/constants.rb

@@ -0,0 +1,35 @@
+module RDB
+  module Length
+    BITS_6 = 0
+    BITS_14 = 1
+    BITS_32 = 2
+    ENCODED = 3
+  end
+
+  module Encoding
+    INT8 = 0
+    INT16 = 1
+    INT32 = 2
+    LZF = 3
+  end
+
+  module Opcode
+    EXPIRETIME_MS = 252
+    EXPIRETIME = 253
+    SELECTDB = 254
+    EOF = 255
+  end
+
+  module Type
+    STRING = 0
+    LIST = 1
+    SET = 2
+    ZSET = 3
+    HASH = 4
+    HASH_ZIPMAP = 9
+    LIST_ZIPLIST = 10
+    SET_INTSET = 11
+    ZSET_ZIPLIST = 12
+    HASH_ZIPLIST = 13
+  end
+end

lib/rdb/errors.rb

@@ -0,0 +1,4 @@
+module RDB
+  class ReaderError < RuntimeError
+  end
+end

lib/rdb/filters.rb

@@ -0,0 +1,7 @@
+module RDB
+  module ObjectFilter
+    def accept_object?(state)
+      true
+    end
+  end
+end

lib/rdb/lzf.rb

@@ -0,0 +1,45 @@
+module RDB
+  module LZF
+    class << self
+      def decompress(rdb, compressed_length, expected_length)
+        ipos = opos = 0
+        input, output = rdb.read(compressed_length), ' ' * expected_length
+
+        while ipos < compressed_length
+          ctrl = input.getbyte(ipos)
+          ipos += 1
+
+          if ctrl < 32
+            (ctrl + 1).times do
+              output.setbyte(opos, input.getbyte(ipos))
+              ipos += 1
+              opos += 1
+            end
+          else
+            length = ctrl >> 5
+
+            if length == 7
+              length = length + input.getbyte(ipos)
+              ipos += 1
+            end
+
+            reference = opos - ((ctrl & 0x1f) << 8) - input.getbyte(ipos) - 1
+            ipos += 1
+
+            (length + 2).times do
+              output.setbyte(opos, output.getbyte(reference))
+              reference += 1
+              opos += 1
+            end
+          end
+        end
+
+        if opos != expected_length
+          raise Exception, "LZF Decompression error: expected length #{expected_length} does not match #{opos}"
+        end
+
+        output
+      end
+    end
+  end
+end

lib/rdb/reader-state.rb

@@ -0,0 +1,25 @@
+module RDB
+  class ReaderState
+    attr_accessor :database, :key, :type, :expiration, :info
+    attr_reader :callbacks, :filter
+
+    def initialize(callbacks = nil, filter = nil)
+      @callbacks = callbacks || EmptyCallbacks.new
+      @filter = filter
+    end
+
+    def key_expires?
+      !@expiration.nil?
+    end
+
+    def mnemonic_type
+      case @type
+      when Type::STRING then :string
+      when Type::SET, Type::SET_INTSET then :set
+      when Type::LIST, Type::LIST_ZIPLIST then :list
+      when Type::ZSET, Type::ZSET_ZIPLIST then :sortedset
+      when Type::HASH, Type::HASH_ZIPMAP, Type::HASH_ZIPLIST then :hash
+      end
+    end
+  end
+end