HTTP GET response body stream to HTTP POST with Ruby

Question

I am trying to upload a large file and then send this file to a REST endpoint using Ruby. A file can be very large, that is, more than can be stored in memory or even in a temporary file on disk. I tried this with Net :: HTTP, but I'm open to solutions with any other library (rest-client, etc.), while they do what I'm trying to do.

Here is what I tried:

require 'net/http'

source_uri = URI("https://example.org/very_large_file")
source_request = Net::HTTP::Get.new(source_uri)
source_http = Net::HTTP.start(source_uri.host, source_uri.port, use_ssl: source_uri.scheme == 'https')

target_uri = URI("https://example2.org/rest/resource")
target_request = Net::HTTP::Post.new(target_uri)
target_http = Net::HTTP.start(target_uri.host, target_uri.port, use_ssl: target_uri.scheme == 'https')

source_response = source_http.request(source_request)
target_request.body = source_response.read_body
target_request.content_type = 'multipart/form-data'
target_response = target_http.request(target_request)

What I want to do is source_response.read_body to return a stream, which I can then pass to target_request in pieces.

Answer 1

: . , Net:: HTTP, , . , .

require 'net/http'
require 'excon'

# provide access to the actual socket
class Net::HTTPResponse
  attr_reader :socket
end

source_uri = URI("https://example.org/very_large_file")
target_uri = URI("https://example2.org/rest/resource")

Net::HTTP.start(source_uri.host, source_uri.port, use_ssl: source_uri.scheme == 'https') do |http|
  request = Net::HTTP::Get.new source_uri

  http.request request do |response|
    len = response.content_length
    p "reading #{len} bytes..."
    read_bytes = 0
    chunk = ''

    chunker = lambda do
      begin
        if read_bytes + Excon::CHUNK_SIZE < len
          chunk = response.socket.read(Excon::CHUNK_SIZE).to_s
          read_bytes += chunk.size
        else
          chunk = response.socket.read(len - read_bytes)
          read_bytes += chunk.size
        end
      rescue EOFError
        # ignore eof
      end
      p "read #{read_bytes} bytes"
      chunk
    end

    Excon.ssl_verify_peer = false
    Excon.post(target_uri.to_s, :request_block => chunker)

  end
end

Answer 2

excon rest-client gem .

, rest-client - multipart/form-data excon, .

, .

require 'excon'
require 'rest-client'

streamer = lambda do |chunk, remaining_bytes, total_bytes|
  puts "Remaining: #{remaining_bytes.to_f / total_bytes}%"
  puts RestClient.post('http://posttestserver.com/post.php', :param1 => chunk)
end

Excon.get('http://textfiles.com/computers/ami-chts.txt', :response_block => streamer)

, ( , , . , , http )

require 'excon'
require 'uri'
require 'net/http'

class Producer
  def initialize
   @mutex = Mutex.new
   @body = ''
  end

  def read(size, out=nil)
    length = nil

    @mutex.synchronize {
      length = @body.slice!(0,size)
    }

    return nil if length.nil? || length.empty?
    out << length if out

    length
  end

  def produce(str)
    @mutex.synchronize {
      @body << str
    }
  end
end

@stream = Producer.new

uri = URI("yourpostaddresshere")
conn = Net::HTTP.new(uri.host, uri.port)
request = Net::HTTP::Post.new uri.request_uri, {'Transfer-Encoding' => 'chunked', 'content-type' => 'text/plain'}
request.body_stream = @stream

Thread.new {
  streamer = lambda do |chunk, remaining_bytes, total_bytes|
    @stream.produce(chunk) 
  end

  Excon.get('http://textfiles.com/computers/ami-chts.txt', :response_block => streamer)
}

conn.start do |http|
  http.request(request)
end

, , HTTP.start (Ruby Net: HTTP change).

Answer 3

, . :

begin
  socket1 = TCPSocket.new(host1, port1)
  socket2 = TCPSocket.new(host2, port2)

  socket1.puts('<GET headers>')
  socket2.puts('<POST headers>')

  IO.copy_stream(socket1, socket2)
ensure
  socket1.close
  socket2.close
end

, , SSL ..

Answer 4

- ( Ruby), - FIFO. , .

FIFO , . . , , , , . FIFO , - ( "" , StringIO).

- :

require 'net/http'

def download_and_upload(source_url, dest_url)
  rd, wr = IO.pipe
  begin
    source_uri = URI.parse(source_url)

    Thread.start do
      begin
        Net::HTTP.start(source_uri.host, source_uri.port, use_ssl: source_uri.scheme == 'https') do |http|
          req = Net::HTTP::Get.new(source_uri.request_uri)
          http.request(req) do |resp|
            resp.read_body do |chunk|
              wr.write(chunk)
              wr.flush
            end
          end
        end
      rescue IOError
        # Usually because the writer was closed
      ensure
        wr.close rescue nil
      end
    end

    dest_uri = URI.parse(dest_url)

    Net::HTTP.start(dest_uri.host, dest_uri.port, use_ssl: dest_uri.scheme == 'https') do |http|
      req = Net::HTTP::Post.new(dest_uri.request_uri)
      req.body_stream = rd
      http.request(req)
    end
  ensure
    rd.close rescue nil
    wr.close rescue nil
  end
end

, , .

, . , . ( , , .)