Why Gemfury? Push, build, and install  RubyGems npm packages Python packages Maven artifacts PHP packages Go Modules Debian packages RPM packages NuGet packages

Repository URL to install this package:

Details    
fakes3 / lib / fakes3 / file_store.rb
Size: Mime:
require 'fileutils'
require 'time'
require 'fakes3/s3_object'
require 'fakes3/bucket'
require 'fakes3/rate_limitable_file'
require 'digest/md5'
require 'yaml'

module FakeS3
  class FileStore
    SHUCK_METADATA_DIR = ".fakes3_metadataFFF"
    # S3 clients with overly strict date parsing fails to parse ISO 8601 dates
    # without any sub second precision (e.g. jets3t v0.7.2), and the examples
    # given in the official AWS S3 documentation specify three (3) decimals for
    # sub second precision.
    SUBSECOND_PRECISION = 3

    def initialize(root)
      @root = root
      @buckets = []
      @bucket_hash = {}
      Dir[File.join(root,"*")].each do |bucket|
        bucket_name = File.basename(bucket)
        bucket_obj = Bucket.new(bucket_name,Time.now,[])
        @buckets << bucket_obj
        @bucket_hash[bucket_name] = bucket_obj
      end
    end

    # Pass a rate limit in bytes per second
    def rate_limit=(rate_limit)
      if rate_limit.is_a?(String)
        if rate_limit =~ /^(\d+)$/
          RateLimitableFile.rate_limit = rate_limit.to_i
        elsif rate_limit =~ /^(.*)K$/
          RateLimitableFile.rate_limit = $1.to_f * 1000
        elsif rate_limit =~ /^(.*)M$/
          RateLimitableFile.rate_limit = $1.to_f * 1000000
        elsif rate_limit =~ /^(.*)G$/
          RateLimitableFile.rate_limit = $1.to_f * 1000000000
        else
          raise "Invalid Rate Limit Format: Valid values include (1000,10K,1.1M)"
        end
      else
        RateLimitableFile.rate_limit = nil
      end
    end

    def buckets
      @buckets
    end

    def get_bucket_folder(bucket)
      File.join(@root,bucket.name)
    end

    def get_bucket(bucket)
      @bucket_hash[bucket]
    end

    def create_bucket(bucket)
      FileUtils.mkdir_p(File.join(@root,bucket))
      bucket_obj = Bucket.new(bucket,Time.now,[])
      if !@bucket_hash[bucket]
        @buckets << bucket_obj
        @bucket_hash[bucket] = bucket_obj
      end
      bucket_obj
    end

    def delete_bucket(bucket_name)
      bucket = get_bucket(bucket_name)
      raise NoSuchBucket if !bucket
      raise BucketNotEmpty if bucket.objects.count > 0
      FileUtils.rm_r(get_bucket_folder(bucket))
      @bucket_hash.delete(bucket_name)
    end

    def get_object(bucket,object_name, request)
      begin
        real_obj = S3Object.new
        obj_root = File.join(@root,bucket,object_name,SHUCK_METADATA_DIR)
        metadata = YAML.load(File.open(File.join(obj_root,"metadata"),'rb'))
        real_obj.name = object_name
        real_obj.md5 = metadata[:md5]
        real_obj.content_type = metadata.fetch(:content_type) { "application/octet-stream" }
        #real_obj.io = File.open(File.join(obj_root,"content"),'rb')
        real_obj.io = RateLimitableFile.open(File.join(obj_root,"content"),'rb')
        real_obj.size = metadata.fetch(:size) { 0 }
        real_obj.creation_date = File.ctime(obj_root).utc.iso8601(SUBSECOND_PRECISION)
        real_obj.modified_date = metadata.fetch(:modified_date) do
          File.mtime(File.join(obj_root,"content")).utc.iso8601(SUBSECOND_PRECISION)
        end
        real_obj.custom_metadata = metadata.fetch(:custom_metadata) { {} }
        return real_obj
      rescue
        puts $!
        $!.backtrace.each { |line| puts line }
        return nil
      end
    end

    def object_metadata(bucket,object)
    end

    def copy_object(src_bucket_name,src_name,dst_bucket_name,dst_name,request)
      src_root = File.join(@root,src_bucket_name,src_name,SHUCK_METADATA_DIR)
      src_metadata_filename = File.join(src_root,"metadata")
      src_metadata = YAML.load(File.open(src_metadata_filename,'rb').read)
      src_content_filename = File.join(src_root,"content")

      dst_filename= File.join(@root,dst_bucket_name,dst_name)
      FileUtils.mkdir_p(dst_filename)

      metadata_dir = File.join(dst_filename,SHUCK_METADATA_DIR)
      FileUtils.mkdir_p(metadata_dir)

      content = File.join(metadata_dir,"content")
      metadata = File.join(metadata_dir,"metadata")

      if src_bucket_name != dst_bucket_name || src_name != dst_name
        File.open(content,'wb') do |f|
          File.open(src_content_filename,'rb') do |input|
            f << input.read
          end
        end

        File.open(metadata,'w') do |f|
          File.open(src_metadata_filename,'r') do |input|
            f << input.read
          end
        end
      end

      metadata_directive = request.header["x-amz-metadata-directive"].first
      if metadata_directive == "REPLACE"
        metadata_struct = create_metadata(content,request)
        File.open(metadata,'w') do |f|
          f << YAML::dump(metadata_struct)
        end
      end

      src_bucket = self.get_bucket(src_bucket_name)
      dst_bucket = self.get_bucket(dst_bucket_name)

      obj = S3Object.new
      obj.name = dst_name
      obj.md5 = src_metadata[:md5]
      obj.content_type = src_metadata[:content_type]
      obj.size = src_metadata[:size]
      obj.modified_date = src_metadata[:modified_date]

      src_obj = src_bucket.find(src_name)
      dst_bucket.add(obj)
      return obj
    end

    def store_object(bucket,object_name,request)
      begin
        filename = File.join(@root,bucket.name,object_name)
        FileUtils.mkdir_p(filename)

        metadata_dir = File.join(filename,SHUCK_METADATA_DIR)
        FileUtils.mkdir_p(metadata_dir)

        content = File.join(filename,SHUCK_METADATA_DIR,"content")
        metadata = File.join(filename,SHUCK_METADATA_DIR,"metadata")

        # TODO put a tmpfile here first and mv it over at the end

        match=request.content_type.match(/^multipart\/form-data; boundary=(.+)/)
      	boundary = match[1] if match
        if boundary
          boundary = WEBrick::HTTPUtils::dequote(boundary)
          filedata = WEBrick::HTTPUtils::parse_form_data(request.body, boundary)
          raise HTTPStatus::BadRequest if filedata['file'].empty?
          File.open(content, 'wb') do |f|
            f << filedata['file']
          end
        else
          File.open(content,'wb') do |f|
            request.body do |chunk|
              f << chunk
            end
          end
        end
        metadata_struct = create_metadata(content,request)

        File.open(metadata,'w') do |f|
          f << YAML::dump(metadata_struct)
        end

        obj = S3Object.new
        obj.name = object_name
        obj.md5 = metadata_struct[:md5]
        obj.content_type = metadata_struct[:content_type]
        obj.size = metadata_struct[:size]
        obj.modified_date = metadata_struct[:modified_date]

        bucket.add(obj)
        return obj
      rescue
        puts $!
        $!.backtrace.each { |line| puts line }
        return nil
      end
    end

    def delete_object(bucket,object_name,request)
      begin
        filename = File.join(@root,bucket.name,object_name)
        FileUtils.rm_rf(filename)
        object = bucket.find(object_name)
        bucket.remove(object)
      rescue
        puts $!
        $!.backtrace.each { |line| puts line }
        return nil
      end
    end

    def delete_objects(bucket, objects, request)
      begin
        filenames = []
        objects.each do |object_name|
          filenames << File.join(@root,bucket.name,object_name)
          object = bucket.find(object_name)
          bucket.remove(object)
        end

        FileUtils.rm_rf(filenames)
      rescue 
        puts $!
        $!.backtrace.each { |line| puts line }
        return nil
      end
    end

    def create_metadata(content,request)
      metadata = {}
      metadata[:md5] = Digest::MD5.file(content).hexdigest
      metadata[:content_type] = request.header["content-type"].first
      metadata[:size] = File.size(content)
      metadata[:modified_date] = File.mtime(content).utc.iso8601(SUBSECOND_PRECISION)
      metadata[:custom_metadata] = {}

      # Add custom metadata from the request header
      request.header.each do |key, value|
        match = /^x-amz-meta-(.*)$/.match(key)
        if match && (match_key = match[1])
          metadata[:custom_metadata][match_key] = value.join(', ')
        end
      end
      return metadata
    end
  end
end