lib/spidr/agent.rb
def visit_page(url)
      url = sanitize_url(url)

      get_page(url) do |page|
        @history << page.url

        begin
          @every_page_blocks.each { |page_block| page_block.call(page) }

          yield page if block_given?
        rescue Actions::Paused => action
          raise(action)
        rescue Actions::SkipPage
          return nil
        rescue Actions::Action
        end

        page.each_url do |next_url|
          begin
            @every_link_blocks.each do |link_block|
              link_block.call(page.url,next_url)
            end
          rescue Actions::Paused => action
            raise(action)
          rescue Actions::SkipLink
            next
          rescue Actions::Action
          end

          if (@max_depth.nil? || @max_depth > @levels[url])
            enqueue(next_url,@levels[url] + 1)
          end
        end
      end
    end
def visit_page(url)
      url = sanitize_url(url)

      get_page(url) do |page|
        @history << page.url

        begin
          @every_page_blocks.each { |page_block| page_block.call(page) }

          yield page if block_given?
        rescue Actions::Paused => action
          raise(action)
        rescue Actions::SkipPage
          return nil
        rescue Actions::Action
        end

        page.each_url do |next_url|
          begin
            @every_link_blocks.each do |link_block|
              link_block.call(page.url,next_url)
            end
          rescue Actions::Paused => action
            raise(action)
          rescue Actions::SkipLink
            next
          rescue Actions::Action
          end

          if (@max_depth.nil? || @max_depth > @levels[url])
            enqueue(next_url,@levels[url] + 1)
          end
        end
      end
    end
def visit_page ( url ) url = sanitize_url ( url ) get_page ( url ) do | page | @history << page . url begin @every_page_blocks . each { | page_block | page_block . call ( page ) } yield page if block_given? rescue Actions :: Paused => action raise ( action ) rescue Actions :: SkipPage return nil rescue Actions :: Action end page . each_url do | next_url | begin @every_link_blocks . each do | link_block | link_block . call ( page . url , next_url ) end rescue Actions :: Paused => action raise ( action ) rescue Actions :: SkipLink next rescue Actions :: Action end if ( @max_depth . nil? || @max_depth > @levels [ url ] ) enqueue ( next_url , @levels [ url ] + 1 ) end end end end
Visits a given URL, and enqueus the links recovered from the URL
 to be visited later.

 @param [URI::HTTP, String] url
   The URL to visit.

 @yield [page]
   If a block is given, it will be passed the page which was visited.

 @yieldparam [Page] page
   The page which was visited.

 @return [Page, nil]
   The page that was visited. If `nil` is returned, either the request
   for the page failed, or the page was skipped.
Visits a given URL and enqueus the links recovered from the URL to be visited later .