from inspect import currentframe, getframeinfo import logging import sys from urllib.error import URLError, HTTPError import urllib.parse import urllib.request import yaml def getLogger(is_debug): logger = logging.getLogger(__name__) handler = logging.StreamHandler(sys.stdout) if (is_debug): logging.basicConfig(level=logging.DEBUG) handler.setLevel(logging.DEBUG) else: logging.basicConfig(level=logging.INFO) handler.setLevel(logging.INFO) formatter = logging.Formatter( fmt='%(asctime)s, %(levelname)-7s [%(filename)s:%(lineno)d, %(funcName)s] %(message)s' ) handler.setFormatter(formatter) logger.propagate = False logger.addHandler(handler) return logger def getFunctionName(): return sys._getframe(1).f_code.co_name # TODO: make output prettier def dumpObject(obj): return yaml.dump(obj) def getCurrentFileName() -> str: return currentframe().f_back.f_code.co_filename def getLineNum() -> int: return currentframe().f_back.f_lineno class URLResponse(object): def __init__(self): self.url:str = '' self.status:int = 0 self.content_type:str = '' self.content_length:int = 0 self.headers:list = [] self.body:bytes = b'\x00' def getURL(url:str, response:URLResponse, logger:logging.Logger=None) -> bool: if not logger: logger = getLogger(True) logger.info(f'fetching URL: {url}') try: res = urllib.request.urlopen(url) except HTTPError as e: logger.error(e) return False except URLError as e: logger.error(e) return False response.url = res.geturl() response.status = res.status response.headers = res.info() response.body = res.read() for k, v in res.info().items(): if k == 'Content-Type': response.content_type = v if k == 'Content-Length': response.content_length = v if response.status != 200 or response.content_length is 0: logger.warning(f'received response for: {url}, status: {response.status}' f' content_length: {response.content_length}') return False return True