#!/usr/bin/python3

from bs4 import BeautifulSoup
import requests


def main(url):
    if not url.startswith("http"):
        url = "http://" + url
    if not url.endswith("/"):
        url += "/"
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    for link in soup.find_all("a", "link-browse"):
        href = link["href"]
        try:
            requests.get(url + href)
        except Exception:
            pass


if __name__ == '__main__':
    from pergamon.settings.local import BASE_URL
    main(BASE_URL)