lib/spidr/agent.rb
def visit?(url)
      !visited?(url) &&
       visit_scheme?(url.scheme) &&
       visit_host?(url.host) &&
       visit_port?(url.port) &&
       visit_link?(url.to_s) &&
       visit_url?(url) &&
       visit_ext?(url.path) &&
       robot_allowed?(url.to_s)
    end
def visit?(url)
      !visited?(url) &&
       visit_scheme?(url.scheme) &&
       visit_host?(url.host) &&
       visit_port?(url.port) &&
       visit_link?(url.to_s) &&
       visit_url?(url) &&
       visit_ext?(url.path) &&
       robot_allowed?(url.to_s)
    end
def visit? ( url ) ! visited? ( url ) && visit_scheme? ( url . scheme ) && visit_host? ( url . host ) && visit_port? ( url . port ) && visit_link? ( url . to_s ) && visit_url? ( url ) && visit_ext? ( url . path ) && robot_allowed? ( url . to_s ) end
Determines if a given URL should be visited.

 @param [URI::HTTP] url
   The URL in question.

 @return [Boolean]
   Specifies whether the given URL should be visited.
Determines if a given URL should be visited .