Pipe data from HTTP GET to HTTP POST/PUT - ruby

Question

I'd like to stream data from an HTTP GET request to an HTTP POST or PUT request. I'd prefer to use Ruby and have already made an attempt using EventMachine and EM-HTTP-Request.
Here's my attempt, to be called using:
HttpToS3Stream.new(src_url, dest_bucket, dest_key, aws_access_key_id, aws_secret_access_key)
http_to_s3_stream.rb
require 'em-http-request'
class HttpToS3Stream
def initialize(http_url, s3_bucket, s3_key, s3_access_key_id, s3_secret_access_key)
#http_url = http_url
#s3_bucket = s3_bucket
#s3_key = s3_key
#s3_access_key_id = s3_access_key_id
#s3_secret_access_key = s3_secret_access_key
go
end
private
def go
EM.run {
# initialize get stream, without listener does not start request
#get_stream = HttpGetStream.new(#http_url)
# initialize put stream, send content length, request starts
#put_stream = S3PutStream.new(#s3_bucket, #s3_key, #s3_access_key_id, #s3_secret_access_key, #get_stream.content_length)
# set listener on get stream, starts request, pipes data to put stream
#get_stream.listener = #put_stream
}
end
end
http_get_stream.rb
require 'httparty'
require 'em-http-request'
class HttpGetStream
def initialize(http_url, listener = nil)
#http_url = http_url
self.listener = listener
end
def listener=(listener)
#listener = listener
listen unless #listener.nil?
end
def content_length
response = HTTParty.head(#http_url)
response['Content-length']
end
private
def listen
http = EventMachine::HttpRequest.new(#http_url).get
http.stream do |chunk|
#listener.send_data chunk
end
http.callback do |chunk|
EventMachine.stop
end
end
end
s3_put_stream.rb
require 'em-http-request'
class S3PutStream
def initialize(s3_bucket, s3_key, s3_access_key_id, s3_secret_access_key, content_length = nil)
#s3_bucket = s3_bucket
#s3_key = s3_key
#s3_access_key_id = s3_access_key_id
#s3_secret_access_key = s3_secret_access_key
#content_length = content_length
#bytes_sent = 0
listen
end
def send_data(data)
#bytes_sent += data.length
#http.on_body_data data
end
private
def listen
raise 'ContentLengthRequired' if #content_length.nil?
#http = EventMachine::HttpRequest.new(put_url).put(
:head => {
'Content-Length' => #content_length,
'Date' => Time.now.getutc,
'Authorization' => auth_key
}
)
#http.errback { |error| puts "error: #{error}" }
end
def put_url
"http://#{#s3_bucket}.s3.amazonaws.com/#{#s3_key}"
end
def auth_key
"#{#s3_access_key_id}:#{#s3_secret_access_key}"
end
end
HttpToS3Stream.new(src_url, dest_bucket, dest_key, aws_access_key_id, aws_secret_access_key)
It seems to be working but always stops at 33468 bytes. Not sure what that's about. Now, by passing chunks directly to #listener.send_data, it is processing the entire GET body. However, the upload is not occurring successfully.
How can I get this to work? And is there a name for what I'm trying to do? I'm having trouble searching for more information.
Any help is appreciated.