-
Notifications
You must be signed in to change notification settings - Fork 1
/
physicstoday_tester.py
executable file
·72 lines (64 loc) · 2.34 KB
/
physicstoday_tester.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#!/usr/bin/env python
from getphysicstodayinfo import getphysicstodayinfo
import cookielib, urllib2
url = 'http://physicstoday.scitation.org/doi/full/10.1063/PT.3.3536'
url = 'http://physicstoday.scitation.org/doi/full/10.1063/1.4796747'
id = url
html = ''
# HTTP request for a webpage URL
try:
## Cookies Support
cookies = cookielib.LWPCookieJar()
handlers = [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
## Add headers to HTTP request so don't get 403 error
hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}
request = urllib2.Request(id, headers=hdr)
html = opener.open(request).read()
urlpage = id # For compatibility down lower in the code
except urllib2.HTTPError, e:
print e.code
html = e.read()
urlpage = id
servererr = True
except:
# Hm...didn't open, and not a 404 or something; try adding http://
if (id.startswith('http://') is False):
urlpage = 'http://' + id
try:
html = urllib3.urlopen(urlpage).read()
except:
# Ok...try adding www. if it's not there
if (urlpage.startswith('http://www.') is False):
urlpage = 'http://www.' + id
try:
html = urllib3.urlopen(urlpage).read()
except:
html = '<html><head>' + \
'<title>BAD LINK: ' + id + \
'</title>' + \
'</head><body></body></html>'
servererr = True
else:
html = '<html><head>' + \
'<title>BAD LINK: ' + id + \
'</title>' + \
'</head><body></body></html>'
paper = getphysicstodayinfo(url, html)
print(paper.title + '\n')
print(paper.author + '\n')
print(paper.date + '\n')
print(paper.abstract + '\n')
print(paper.sources + '\n')
print(paper.subject + '\n')
print(paper.url + '\n')
print(paper.errors + '\n')