src/naarad/httpdownload.py
def get_urls_from_seed(url):
  """
  get a list of urls from a seeding url, return a list of urls

  :param str url: a full/absolute url, e.g. http://www.cnn.com/logs/
  :return: a list of full/absolute urls.
  """

  if not url or type(url) != str or not naarad.utils.is_valid_url(url):
    logger.error("get_urls_from_seed() does not have valid seeding url.")
    return

  # Extract the host info of "http://host:port/" in case of href urls are elative urls (e.g., /path/gc.log)
  # Then join (host info and relative urls) to form the complete urls
  base_index = url.find('/', len("https://"))   # get the first "/" after http://" or "https://"; handling both cases.
  base_url = url[:base_index]      # base_url = "http://host:port" or https://host:port" or http://host" (where no port is given)

  # Extract the "href" denoted urls
  urls = []
  try:
    response = urllib2.urlopen(url)
    hp = HTMLLinkExtractor()
    hp.feed(response.read())
    urls = hp.links
    hp.close()
  except urllib2.HTTPError:
    logger.error("Got HTTPError when opening the url of %s" % url)
    return urls

  # Check whether the url is relative or complete
  for i in range(len(urls)):
    if not urls[i].startswith("http://") and not urls[i].startswith("https://"):    # a relative url ?
      urls[i] = base_url + urls[i]

  return urls
def get_urls_from_seed(url):
  """
  get a list of urls from a seeding url, return a list of urls

  :param str url: a full/absolute url, e.g. http://www.cnn.com/logs/
  :return: a list of full/absolute urls.
  """

  if not url or type(url) != str or not naarad.utils.is_valid_url(url):
    logger.error("get_urls_from_seed() does not have valid seeding url.")
    return

  # Extract the host info of "http://host:port/" in case of href urls are elative urls (e.g., /path/gc.log)
  # Then join (host info and relative urls) to form the complete urls
  base_index = url.find('/', len("https://"))   # get the first "/" after http://" or "https://"; handling both cases.
  base_url = url[:base_index]      # base_url = "http://host:port" or https://host:port" or http://host" (where no port is given)

  # Extract the "href" denoted urls
  urls = []
  try:
    response = urllib2.urlopen(url)
    hp = HTMLLinkExtractor()
    hp.feed(response.read())
    urls = hp.links
    hp.close()
  except urllib2.HTTPError:
    logger.error("Got HTTPError when opening the url of %s" % url)
    return urls

  # Check whether the url is relative or complete
  for i in range(len(urls)):
    if not urls[i].startswith("http://") and not urls[i].startswith("https://"):    # a relative url ?
      urls[i] = base_url + urls[i]

  return urls
def get_urls_from_seed ( url ) : if not url or type ( url ) != str or not naarad . utils . is_valid_url ( url ) : logger . error ( "get_urls_from_seed() does not have valid seeding url." ) return # Extract the host info of "http://host:port/" in case of href urls are elative urls (e.g., /path/gc.log) # Then join (host info and relative urls) to form the complete urls base_index = url . find ( '/' , len ( "https://" ) ) # get the first "/" after http://" or "https://"; handling both cases. base_url = url [ : base_index ] # base_url = "http://host:port" or https://host:port" or http://host" (where no port is given) # Extract the "href" denoted urls urls = [ ] try : response = urllib2 . urlopen ( url ) hp = HTMLLinkExtractor ( ) hp . feed ( response . read ( ) ) urls = hp . links hp . close ( ) except urllib2 . HTTPError : logger . error ( "Got HTTPError when opening the url of %s" % url ) return urls # Check whether the url is relative or complete for i in range ( len ( urls ) ) : if not urls [ i ] . startswith ( "http://" ) and not urls [ i ] . startswith ( "https://" ) : # a relative url ? urls [ i ] = base_url + urls [ i ] return urls
get a list of urls from a seeding url, return a list of urls

  :param str url: a full/absolute url, e.g. http://www.cnn.com/logs/
  :return: a list of full/absolute urls.
get a list of urls from a seeding url return a list of urls