You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
85 lines
2.2 KiB
85 lines
2.2 KiB
|
|
from inspect import currentframe, getframeinfo |
|
import logging |
|
import sys |
|
from urllib.error import URLError, HTTPError |
|
import urllib.parse |
|
import urllib.request |
|
import yaml |
|
|
|
|
|
def getLogger(is_debug): |
|
logger = logging.getLogger(__name__) |
|
handler = logging.StreamHandler(sys.stdout) |
|
|
|
if (is_debug): |
|
logging.basicConfig(level=logging.DEBUG) |
|
handler.setLevel(logging.DEBUG) |
|
else: |
|
logging.basicConfig(level=logging.INFO) |
|
handler.setLevel(logging.INFO) |
|
|
|
formatter = logging.Formatter( |
|
fmt='%(asctime)s, %(levelname)-7s [%(filename)s:%(lineno)d, %(funcName)s] %(message)s' |
|
) |
|
|
|
handler.setFormatter(formatter) |
|
logger.propagate = False |
|
logger.addHandler(handler) |
|
return logger |
|
|
|
def getFunctionName(): |
|
return sys._getframe(1).f_code.co_name |
|
|
|
# TODO: make output prettier |
|
def dumpObject(obj): |
|
return yaml.dump(obj) |
|
|
|
def getCurrentFileName() -> str: |
|
return currentframe().f_back.f_code.co_filename |
|
|
|
def getLineNum() -> int: |
|
return currentframe().f_back.f_lineno |
|
|
|
|
|
class URLResponse(object): |
|
def __init__(self): |
|
self.url:str = '' |
|
self.status:int = 0 |
|
self.content_type:str = '' |
|
self.content_length:int = 0 |
|
self.headers:list = [] |
|
self.body:bytes = b'\x00' |
|
|
|
def getURL(url:str, response:URLResponse, logger:logging.Logger=None) -> bool: |
|
if not logger: |
|
logger = getLogger(True) |
|
|
|
logger.info(f'fetching URL: {url}') |
|
|
|
try: |
|
res = urllib.request.urlopen(url) |
|
except HTTPError as e: |
|
logger.error(e) |
|
return False |
|
except URLError as e: |
|
logger.error(e) |
|
return False |
|
|
|
response.url = res.geturl() |
|
response.status = res.status |
|
response.headers = res.info() |
|
response.body = res.read() |
|
|
|
for k, v in res.info().items(): |
|
if k == 'Content-Type': |
|
response.content_type = v |
|
if k == 'Content-Length': |
|
response.content_length = v |
|
|
|
if response.status != 200 or response.content_length is 0: |
|
logger.warning(f'received response for: {url}, status: {response.status}' |
|
f' content_length: {response.content_length}') |
|
return False |
|
|
|
return True
|
|
|