import glob
import time
import urllib.error
import urllib.request

import requests
import rdflib

HEADERS = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:64.0) Gecko/20100101 Firefox/64.0'}

def find_image(rdf_doc):
    """
    Returns the first dwc:associatedMedia resource in the rdf_doc
    """
    graph = rdflib.Graph()
    graph.parse(rdf_doc)
    mediaref = rdflib.URIRef("http://rs.tdwg.org/dwc/terms/associatedMedia")
    for subj, pred, obj in graph:
        if pred == mediaref:
            return obj


with open("belgian_colony_data_all.csv") as bcfile:
    for line in bcfile:
        barcode = line.split(",")[0]

        # skip files we already have already scraped
        if glob.glob("specimen_img_raw/" + barcode + "*"):
            print("we already have", barcode, "skipping to...")
            continue

        rdf_url = "https://www.botanicalcollections.be/specimen/" + barcode + "/rdf"
        print("fetching", rdf_url)
        try:
            img_url = find_image(rdf_url)
        except urllib.error.HTTPError as er:
            print("Error on ", rdf_url, str(er))
            continue
        if img_url is None:
            print("Error. No image for barcode.")
            continue
        img_ext = str(img_url).rsplit(".", 1)[1]
        print("image url:", img_url)
        save_path = "specimen_img_raw/" + barcode + "." + img_ext
        save_resp = requests.get(img_url, headers=HEADERS, allow_redirects=True)
        open(save_path, "wb").write(save_resp.content)
        time.sleep(1)