Upload files in AWS S3 Glacier in Rails
How to upload a file using multipart to glacier in Ruby
What is Amazon Glacier?
Amazon Glacier is a storage optimized for infrequently used data, or cold data. It is an extremely low-cost storage service that provides durable storage with security features for data archiving and backup.
# Gemfile
gem 'aws-sdk', '~> 2.3'
gem 'aws-sdk', '~> 2.3'
Run bundle install and restart the Rails server after modifying the Gemfile.
View Page :-
<%= form_tag buy_glacier_upload_path, method: :post, multipart: true do %>
<%= file_field_tag :file, required: true %>
<%= submit_tag "Upload File", class: "btn submit-btn" %>
<% end %>
Now in the controller :-
class HomesController < ApplicationController
ACCOUNT_ID = "6363463563636536"
AWS_ACCESS_ID = "HDGHDHDHHFGHRGF"
AWS_SECRET_KEY = "Dghghghgggggggggghghtyeryey+/"
AWS_REGION = "ap-southeast-2"
AWS_VAULT = "videos"
def glacier_upload
uploaded_file = params[:file]
glacier_client = Aws::Glacier::Client.new(region: AWS_REGION, access_key_id: AWS_ACCESS_ID, secret_access_key: AWS_SECRET_KEY)
file_path = uploaded_file.path
file = File.open(file_path, "r")
source_file = file.read
file_size = file.size
original_filename = uploaded_file.original_filename
response = glacier_client.initiate_multipart_upload({account_id: ACCOUNT_ID, part_size: 2097152, vault_name: AWS_VAULT })
upload_id = response.to_h[:upload_id]
puts "Uploaded id #{upload_id}"
total_checksum = Treehash::calculate_tree_hash source_file
resp = glacier_client.upload_archive({
account_id: ACCOUNT_ID,
archive_description: "Testing from RY",
body: source_file,
checksum: total_checksum,
vault_name: AWS_VAULT
})
# split the file into chunks with the size same as range
File.open(file_path, "rb") do |f|
start_byte = 0
end_byte = 2097151
f.each_chunk(2097152) do |chunk|
# last chunk's end_byte can be less than the part_size
if end_byte > file_size
end_byte = file_size - 1
end
# generate checksum for each chunk
chunk_checksum = Treehash::calculate_tree_hash chunk
resp = glacier_client.upload_multipart_part({
account_id: ACCOUNT_ID,
vault_name: AWS_VAULT,
upload_id: upload_id,
checksum: chunk_checksum,
range: "bytes #{start_byte}-#{end_byte}/*",
body: chunk
})
start_byte = start_byte + 2097152
end_byte = end_byte + 2097152
end
end
resp = glacier_client.upload_multipart_part({
account_id: ACCOUNT_ID,
body: source_file,
checksum: total_checksum,
range: "bytes 0-#{file_size-1}/*",
upload_id: upload_id,
vault_name: AWS_VAULT
})
resp = glacier_client.complete_multipart_upload({
account_id: ACCOUNT_ID,
archive_size: file_size,
checksum: total_checksum,
upload_id: upload_id,
vault_name: AWS_VAULT
})
archive_id = resp.to_h[:archive_id]
location = resp.to_h[:location]
checksum = resp.to_h[:checksum]
if archive_id.present?
puts "Successfully uploaded, Archive id is #{archive_id}"
puts "Successfully uploaded, Location is #{location}"
puts "Successfully uploaded, Checksum is #{checksum}"
else
puts "Please Try again."
end
redirect_to root_path
end
end
Test the uploaded files in rails console :-
resp = glacier_client.list_jobs({ account_id: ACCOUNT_ID, vault_name: AWS_VAULT })
resp = glacier_client.describe_vault({ account_id: ACCOUNT_ID, vault_name: AWS_VAULT })
resp = glacier_client.list_jobs({ account_id: ACCOUNT_ID, vault_name: AWS_VAULT })
resp.job_list
resp.job_list[0].to_h[:job_id]
resp.job_list.count = 12
resp = glacier_client.describe_job({ account_id: ACCOUNT_ID, vault_name: AWS_VAULT, job_id: "fdfdasfafafasfas9496456445gdfgdfgdsqjhgjgj-etY^TRwetrwetee" })
Also, see :
https://docs.aws.amazon.com/sdkforruby/api/Aws/Glacier/Client.html#upload_multipart_part-instance_method
https://www.inkoop.io/blog/upload-a-file-using-multipart-to-glacier-in-ruby/
Comments