-
Notifications
You must be signed in to change notification settings - Fork 2
/
get_ems_zips.py
54 lines (35 loc) · 1.46 KB
/
get_ems_zips.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
#this script donwload ZIP files of EMS setting a list of EMS tags in the emergency_tags (see example)
#!/usr/bin/env python
import requests
import html5lib
import os
import subprocess
from bs4 import BeautifulSoup
def get_zip_links(archive_url):
# create response object
r = requests.get(archive_url)
# create beautiful-soup object
soup = BeautifulSoup(r.content,'html5lib')
# find all links on web-page
links = soup.findAll('a')
# filter the link sending with .zip
zip_links = ["http://emergency.copernicus.eu" + link['href'] for link in links if link['href'].endswith('zip')]
return zip_links
def download_zip_series(zip_links):
#iterate through all links in zip_links and download them one by one
for link in zip_links:
#obtain filename
file_name = link.split('/')[-1]
command="curl -# "+'"'+link+'" -H "User-Agent: EMS Toolkit (dfe3b314c95db26fafbc758c17db76ec)" -H "Referer: '+link+'" -H "Connection: keep-alive" > '+ file_name
os.system(command)
print "%s downloaded!\n"%file_name
return
def get_ems_zips (emergency_tags):
for ems in emergency_tags: # main
archive_url = "http://emergency.copernicus.eu/mapping/list-of-components/" + ems
# getting all zip links
zip_links = get_zip_links(archive_url)
print zip_links
# download all zip files
download_zip_series(zip_links)
print "Done. All zip files are downloaded!"