S3 File Resource for Chef

recipe.rb

# Source accepts the protocol s3:// with the host as the bucket
# access_key_id and secret_access_key are just that
s3_file "/var/bulk/the_file.tar.gz" do
  source "s3://your.bucket/the_file.tar.gz"
  access_key_id your_key
  secret_access_key your_secret
  owner "root"
  group "root"
  mode 0644
end

s3_file.rb

#
# Author:: Christopher Peplin (<[email protected]>)
# Copyright:: Copyright (c) 2010 Bueda, Inc.
# License:: Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

class Chef
  class Provider
    class S3File < Chef::Provider::RemoteFile
      def action_create
        Chef::Log.debug("Checking #{@new_resource} for changes")

        if current_resource_matches_target_checksum?
          Chef::Log.debug("File #{@new_resource} checksum matches target checksum (#{@new_resource.checksum}), not updating")
        else
          Chef::Log.debug("File #{@current_resource} checksum didn't match target checksum (#{@new_resource.checksum}), updating")
          fetch_from_s3(@new_resource.source) do |raw_file|
            if matches_current_checksum?(raw_file)
              Chef::Log.debug "#{@new_resource}: Target and Source checksums are the same, taking no action"
            else
              backup_new_resource
              Chef::Log.debug "copying remote file from origin #{raw_file.path} to destination #{@new_resource.path}"
              FileUtils.cp raw_file.path, @new_resource.path
              @new_resource.updated = true
            end
          end
        end
        enforce_ownership_and_permissions

        @new_resource.updated
      end

      def fetch_from_s3(source)
        begin
          protocol, bucket, name = URI.split(source).compact
          name = name[1..-1]
          AWS::S3::Base.establish_connection!(
              :access_key_id     => @new_resource.access_key_id,
              :secret_access_key => @new_resource.secret_access_key
          )
          obj = AWS::S3::S3Object.find name, bucket
          Chef::Log.debug("Downloading #{name} from S3 bucket #{bucket}")
          file = Tempfile.new("chef-s3-file")
          file.write obj.value
          Chef::Log.debug("File #{name} is #{file.size} bytes on disk")
          begin
            yield file
          ensure
            file.close
          end
        rescue URI::InvalidURIError
          Chef::Log.warn("Expected an S3 URL but found #{source}")
          nil
        end
      end
    end
  end
end

class Chef
  class Resource
    class S3File < Chef::Resource::RemoteFile
      def initialize(name, run_context=nil)
        super
        @resource_name = :s3_file
      end

      def provider
        Chef::Provider::S3File
      end

      def access_key_id(args=nil)
        set_or_return(
          :access_key_id,
          args,
          :kind_of => String
        )
      end
        
      def secret_access_key(args=nil)
        set_or_return(
          :secret_access_key,
          args,
          :kind_of => String
        )
      end
    end 
  end
end

These all have the same problem - to be generally useful, the temporary file needs to be written in binary mode! e.g. for the above:

file = Tempfile.new("chef-s3-file")
file.binmode
file.write obj.value

Getting a "no resource or method named s3_file for Chef::recipe "default" when we try to use this. ANy ideas?

Have you tried to add include_recipe "s3_file" into your cookbook recipe?

i get "no resource or method named s3_file for Chef::recipe "default" no matter what i try

Hi,

I want how to write (or) the S3 file.

Thanks
Karthi

Can this resource copy a directory of files to a local directory as well as just a single file?

Hello. I am getting the following... any help appreciated:

19: class Chef
20: class Provider
21>> class S3File < Chef::Provider::RemoteFile
22: def action_create
23: Chef::Log.debug("Checking #{@new_resource} for changes")
24:
25: if current_resource_matches_target_checksum?
26: Chef::Log.debug("File #{@new_resource} checksum matches target checksum (#{@new_resource.checksum}), not updating")
27: else
28: Chef::Log.debug("File #{@current_resource} checksum didn't match target checksum (#{@new_resource.checksum}), updating")
29: fetch_from_s3(@new_resource.source) do |raw_file|
30: if matches_current_checksum?(raw_file)

[2016-08-25T09:50:43-04:00] ERROR: Running exception handlers
[2016-08-25T09:50:43-04:00] ERROR: Exception handlers complete
[2016-08-25T09:50:43-04:00] FATAL: Stacktrace dumped to /var/chef/cache/chef-stacktrace.out
Chef Client failed. 0 resources updated
[2016-08-25T09:50:43-04:00] ERROR: uninitialized constant #Class:0x00000003144990::Chef::Provider::RemoteFile
[2016-08-25T09:50:43-04:00] FATAL: Chef::Exceptions::ChildConvergeError: Chef run process exited unsuccessfully (exit code 1)

Hi,
this doesn't work for me when I don't pass access keys, actually the instance already has an IAM role assigned and has enough permission on bucket.