Updates to fix current bugs in the project, and adds an example

examples/example1.py

@@ -10,9 +10,12 @@
   gs.results_per_page = 50
   results = gs.get_results()
   for res in results:
-    print res.title.encode('utf8')
-    print res.desc.encode('utf8')
-    print res.url.encode('utf8')
+    if res.title is not None:
+        print res.title.encode('utf8')
+    if res.desc is not None:
+        print res.desc.encode('utf8')
+    if res.url is not None:
+        print res.url.encode('utf8')
     print
 except SearchError, e:
   print "Search failed: %s" % e

examples/example4.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python
+#
+# Justin Vieira ([email protected])
+# http://www.rancorsoft.com  --  Let's Rock Together.
+#
+# This program does a Google search for "super test results" and returns
+# all results.
+#
+
+from xgoogle.search import GoogleSearch, SearchError
+from threading import Thread
+from random import randint
+import time
+
+try:
+  gs = GoogleSearch("super test results")
+  gs.results_per_page = 50
+  displayedResults = 0
+  results = gs.get_results()
+  while displayedResults < gs.num_results:
+      for res in results:
+        if res.title is not None:
+            print res.title.encode('utf8')
+        if res.desc is not None:
+            print res.desc.encode('utf8')
+        if res.url is not None:
+            print res.url.encode('utf8')
+        displayedResults += gs.results_per_page
+        print
+      time.sleep(randint(15,60))
+      results = gs.get_results()
+except SearchError, e:
+  print "Search failed: %s" % e
+

xgoogle/search.py

@@ -55,7 +55,7 @@ class GoogleSearch(object):
     SEARCH_URL_1 = "http://www.google.%(tld)s/search?hl=%(lang)s&q=%(query)s&num=%(num)d&btnG=Google+Search"
     NEXT_PAGE_1 = "http://www.google.%(tld)s/search?hl=%(lang)s&q=%(query)s&num=%(num)d&start=%(start)d"
 
-    def __init__(self, query, random_agent=False, debug=False, lang="en", tld="com", re_search_strings=None):
+    def __init__(self, query, random_agent=True, debug=False, lang="en", tld="com", re_search_strings=None):
         self.query = query
         self.debug = debug
         self.browser = Browser(debug=debug)
@@ -237,8 +237,8 @@ def _extract_results(self, soup):
     def _extract_result(self, result):
         title, url = self._extract_title_url(result)
         desc = self._extract_description(result)
-        if not title or not url or not desc:
-            return None
+        #if not title or not url or not desc:
+        #    return None
         return SearchResult(title, url, desc)
 
     def _extract_title_url(self, result):
@@ -322,7 +322,8 @@ def _extract_info(self, soup):
         return {'from': int(matches.group(1)), 'to': int(matches.group(2)), 'total': int(matches.group(3))}
 
     def _extract_results(self, soup):
-        results = soup.findAll('p', {'class': 'g'})
+        #results = soup.findAll('p', {'class': 'g'})
+        results = soup.findAll('li','g')
         ret_res = []
         for result in results:
             eres = self._extract_result(result)
@@ -352,7 +353,8 @@ def _extract_title_url(self, result):
         return title, url
 
     def _extract_description(self, result):
-        desc_td = result.findNext('td')
+        #desc_td = result.findNext('td')
+        desc_div = result.find('span', 'st')
         if not desc_td:
             self._maybe_raise(ParseError, "Description tag in Google search result was not found", result)
             return None