lib/spidr/agent.rb
def prepare_request(url,&block)
      path = unless url.path.empty?
               url.path
             else
               '/'
             end

      # append the URL query to the path
      path += "?#{url.query}" if url.query

      headers = prepare_request_headers(url)

      begin
        sleep(@delay) if @delay > 0

        yield @sessions[url], path, headers
      rescue SystemCallError,
             Timeout::Error,
             SocketError,
             IOError,
             OpenSSL::SSL::SSLError,
             Net::HTTPBadResponse,
             Zlib::Error

        @sessions.kill!(url)

        failed(url)
        return nil
      end
    end
def prepare_request(url,&block)
      path = unless url.path.empty?
               url.path
             else
               '/'
             end

      # append the URL query to the path
      path += "?#{url.query}" if url.query

      headers = prepare_request_headers(url)

      begin
        sleep(@delay) if @delay > 0

        yield @sessions[url], path, headers
      rescue SystemCallError,
             Timeout::Error,
             SocketError,
             IOError,
             OpenSSL::SSL::SSLError,
             Net::HTTPBadResponse,
             Zlib::Error

        @sessions.kill!(url)

        failed(url)
        return nil
      end
    end
def prepare_request ( url , & block ) path = unless url . path . empty? url . path else '/' end # append the URL query to the path path += "?#{url.query}" if url . query headers = prepare_request_headers ( url ) begin sleep ( @delay ) if @delay > 0 yield @sessions [ url ] , path , headers rescue SystemCallError , Timeout :: Error , SocketError , IOError , OpenSSL :: SSL :: SSLError , Net :: HTTPBadResponse , Zlib :: Error @sessions . kill! ( url ) failed ( url ) return nil end end
Normalizes the request path and grabs a session to handle page
 get and post requests.

 @param [URI::HTTP] url
   The URL to request.

 @yield [request]
   A block whose purpose is to make a page request.

 @yieldparam [Net::HTTP] session
   An HTTP session object.

 @yieldparam [String] path
   Normalized URL string.

 @yieldparam [Hash] headers
   A Hash of request header options.

 @since 0.2.2
Normalizes the request path and grabs a session to handle page get and post requests .