Added urllib-downloader koubae

.gitignore

@@ -1,4 +1,6 @@
 *.log
 *.json
 data/
-logs/
+logs/
+__pycache__
+.idea

urllib_downloader/Example/multiproc.py

@@ -0,0 +1,42 @@
+import os,sys,inspect
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0,parentdir)
+from download_urllib import download_manager
+
+
+main_keywords = ['neutral', 'angry', 'surprise', 'disgust', 'fear', 'happy', 'sad']
+
+supplemented_keywords = ['facial expression',
+                         'human face',
+                         'face',
+                         'old face',
+                         'young face',
+                         'adult face',
+                         'child face',
+                         'woman face',
+                         'man face',
+                         'male face',
+                         'female face',
+                         'gentleman face',
+                         'lady face',
+                         'boy face',
+                         'girl face',
+                         'American face',
+                         'Chinese face',
+                         'Korean face',
+                         'Japanese face',
+                         'actor face',
+                         'actress face'
+                         'doctor face',
+                         'movie face'
+                         ]
+
+if __name__ == '__main__':
+    # Multiprocess
+    keywords = ['Car', 'Motorbike']
+    extra__words = ['red', 'blue', 'green', 'expensive']
+    download_manager(keywords, extra_keywords=extra__words, total=3, multiprocess=True)
+    download_manager(main_keywords, extra_keywords=supplemented_keywords, total=10,
+                     multiprocess=True, debug=True, download_dir='.new_dir/')
+

urllib_downloader/Example/simple.py

@@ -0,0 +1,18 @@
+import os,sys,inspect
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0,parentdir)
+from download_urllib import download_manager
+
+
+# Simple Download
+main_keywords = ['Pizza']
+download_manager(main_keywords)
+
+
+main_keywords = ['Car', 'Motorbike']
+download_manager(main_keywords, total=3)
+
+main_keywords = ['Car', 'Motorbike']
+extra_words = ['red', 'blue', 'green', 'expensive']
+download_manager(main_keywords, extra_keywords=extra_words, total=3)

urllib_downloader/README.MD

@@ -0,0 +1,48 @@
+Picture Downloader 
+=======================
+
+
+Simple Picture Downloader using Google search and urllib.request
+
+@Author: [lc](https://github.com/WuLC)
+@Last Modified by: [Koubae](https://github.com/Koubae)
+
+-----------------------------------------------------------------------------------------------------
+
+
+REQ REQUIREMENTS:
+-------------------
+
+
+1. **http.client — HTTP protocol client** [DOCS](https://docs.python.org/3/library/http.client.html)
+ - [Source Code](https://github.com/python/cpython/blob/3.9/Lib/http/client.py)
+This module defines classes which implement the client side of the HTTP and 
+HTTPS protocols. It is normally not used directly — the module urllib.request
+
+2. **urllib.request — Extensible library for opening URLs** [DOCS](https://docs.python.org/3/library/urllib.request.html#module-urllib.request)
+
+- [Source Code](https://github.com/python/cpython/blob/3.9/Lib/urllib/request.py)
+
+3. **urllib.parse — Parse URLs into components** [DOCS](https://docs.python.org/3/library/urllib.parse.html#module-urllib.parse)
+
+4. **Regular expression operations re** DOCS](https://docs.python.org/3/library/re.html)
+
+- Google for Education [Python Regular Expressions](https://developers.google.com/edu/python/regular-expressions)
+
+5. **os** [DOCS](https://docs.python.org/3/library/os.html)
+
+6. **logging — Logging facility for Python¶** [DOCS](https://docs.python.org/3/library/logging.html)
+
+7. **multiprocessing — Process-based parallelism** [DOCS](https://docs.python.org/3/library/multiprocessing.html)
+
+### Third party package
+
+1. **user_agent** This module is for generating random, valid web user agents:
+
+- [Source Code](https://github.com/lorien/user_agent)  *install_requires=['six']*
+
+
+
+
+
+

urllib_downloader/Tests/test.py

@@ -0,0 +1,66 @@
+import os,sys,inspect
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+sys.path.insert(0,parentdir)
+from download_urllib import download_manager
+import unittest
+
+# ..urllib_downloader.donwload_urllib
+class Test(unittest.TestCase):
+
+    def test_one_keyword(self):
+        main_keyword = ['neutral']
+        download_manager(main_keyword, extra_keywords=None, total=2)
+
+    def test_download_false(self):
+        main_keyword = ['neutral']
+        download_manager(main_keyword, extra_keywords=None, total=2, download=False)
+
+    def test_custom_dir(self):
+        main_keywords = ['pizza', 'pasta']
+        extra_k = ['pomodoro', 'salami', 'tuna']
+        my_dir = './my_dir/'
+        download_manager(main_keywords, extra_keywords=extra_k, download_dir=my_dir, total=2)
+
+    def test_multiprocess(self):
+        main_keywords = ['neutral', 'angry']
+        supplemented_keywords = ['facial expression', 'people', 'covid', 'world']
+        download_manager(main_keywords, extra_keywords=supplemented_keywords, total=10, multiprocess=True, debug=True)
+
+        main_keywords = ['neutral', 'angry', 'surprise', 'disgust', 'fear', 'happy', 'sad']
+
+        supplemented_keywords = ['facial expression', \
+                                 'human face', \
+                                 'face', \
+                                 'old face', \
+                                 'young face', \
+                                 'adult face', \
+                                 'child face', \
+                                 'woman face', \
+                                 'man face', \
+                                 'male face', \
+                                 'female face', \
+                                 'gentleman face', \
+                                 'lady face', \
+                                 'boy face', \
+                                 'girl face', \
+                                 'American face', \
+                                 'Chinese face', \
+                                 'Korean face', \
+                                 'Japanese face', \
+                                 'actor face', \
+                                 'actress face' \
+                                 'doctor face', \
+                                 'movie face'
+                                 ]
+        download_manager(main_keywords, extra_keywords=supplemented_keywords, total='all', multiprocess=True, debug=True)
+
+
+def run_tests(test_class):
+    suite = unittest.TestLoader().loadTestsFromTestCase(test_class)
+    runner = unittest.TextTestRunner(verbosity=0)
+    result = runner.run(suite)
+
+
+if __name__ == '__main__':
+    run_tests(Test)