add getURL() to util.py

video_metadata/util.py

@@ -2,6 +2,9 @@
 from inspect import currentframe, getframeinfo
 import logging
 import sys
+from urllib.error import URLError, HTTPError
+import urllib.parse
+import urllib.request
 import yaml
 
 
@@ -37,3 +40,46 @@ def getCurrentFileName() -> str:
 
 def getLineNum() -> int:
     return currentframe().f_back.f_lineno
+
+
+class URLResponse(object):
+    def __init__(self):
+        self.url:str = ''
+        self.status:int = 0
+        self.content_type:str = ''
+        self.content_length:int = 0
+        self.headers:list = []
+        self.body:bytes = b'\x00'
+
+def getURL(url:str, response:URLResponse, logger:logging.Logger=None) -> bool:
+    if not logger:
+        logger = getLogger(True)
+
+    logger.info(f'fetching URL: {url}')
+
+    try:
+        res = urllib.request.urlopen(url)
+    except HTTPError as e:
+        logger.error(e)
+        return False
+    except URLError as e:
+        logger.error(e)
+        return False
+
+    response.url = res.geturl()
+    response.status = res.status
+    response.headers = res.info()
+    response.body = res.read()
+
+    for k, v in res.info().items():
+        if k == 'Content-Type':
+            response.content_type = v
+        if k == 'Content-Length':
+            response.content_length = v
+
+    if response.status != 200 or response.content_length is 0:
+        logger.warning(f'received response for: {url}, status: {response.status}'
+                f' content_length: {response.content_length}')
+        return False
+
+    return True