"""File:     url_list.py
   Purpose:  Build a list of the links on a web page.

   Run:      python url_list.py

   Input:    root URL 
             HTML filename

   Output:   List of links on the page.

   Note:     Each line of the HTML file is either a tag
             or a word.
"""
import requests

#----------------------------------------------------------------------
def get_urls(pg_txt, root_url):
   """Extract urls from an html page stored as a string in pg_txt"""
   ulist = []
   pg_strs = pg_txt.split('\n')
   for i in range(len(pg_strs)):
      pg_strs[i] = pg_strs[i].strip()
      if len(pg_strs[i]) >= 1:
         if pg_strs[i][0:2] == "<a":
            url = pg_strs[i][9:len(pg_strs[i])-2]
            url = root_url + '/' + url
            ulist.append(url)
   return ulist

#----------------------------------------------------------------------
def print_url_list(curr_site, ulist):
   print "The URLs on the page", curr_site, "are"
   for url in ulist:
      print "  ", url

#----------------------------------------------------------------------
root_url = raw_input("What's the root URL for the web crawl?\n   ")
rel_site = raw_input("What's the name of the html file\n   ")
current_site = root_url + "/" + rel_site
page = requests.get(current_site)
url_list = get_urls(page.content, root_url)

print_url_list(current_site, url_list)