import cgi from dataclasses import dataclass, field from datetime import date, datetime from enum import IntEnum import json import re from sqlite3 import DatabaseError import ssl from typing import Any, Dict, List from urllib.error import URLError, HTTPError import urllib.parse import urllib.request from flask import Blueprint, current_app, flash, Flask, jsonify, request, render_template, Response from .. import util from .. import db #################### # constants TMDB_API_URL = 'https://api.themoviedb.org' TMDB_SEARCH_MOVIE = '/3/search/movie' TMDB_SEARCH_TV = '/3/search/tv' TMDB_MOVIE_LINK_URL = 'https://www.themoviedb.org/movie/' TMDB_TV_LINK_URL = 'https://www.themoviedb.org/tv/' #################### # globals bp = Blueprint('tmdb', __name__, url_prefix='/tmdb') #################### # data structures class media_type(IntEnum): UNKNOWN = 0 MOVIE = 1 TV_SHOW = 2 @dataclass class query_data: errors: List[str] = field(default_factory=list) full_url: str = '' movie_year: int = 0 parsed_query_str: str = '' query_params: str = '' media_type: media_type = media_type.UNKNOWN raw_query_str: str = '' rel_path: str = '' @dataclass class response_data: errors: List[str] = field(default_factory=list) parsed_response: Any = field(default_factory=object) response_body: str = '' response_headers: List[str] = field(default_factory=list) response_status: int = 0 result_count: int = 0 @dataclass class view_result: media_type:str = 'unknown' id: int = 0 title: str = '' release_date: str = '' overview: str = '' genre_ids: List[int] = field(default_factory=list) original_language: str = '' poster_path: str = 'no_image.png' tmdb_link: str = '' # TODO: flask 1.1 adds support for JSON encoding dataclasses # https://palletsprojects.com/blog/flask-1-1-released/ # add flask 1.1 to requirements when alpine distro updates flask # NOTE: haven't found a better way to json encode dataclasses :( class result_encoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, view_result): return obj.__dict__ if isinstance(obj, date): return obj.isoformat() if isinstance(obj, datetime): return obj.isoformat() try: ret = json.JSONEncoder.default(self, obj) except TypeError: return '' return ret #################### # flask routes @bp.route('/list_movie_test') @bp.route('/list_movie_test/') def list_movie_test(sorting = 'title'): if sorting == 'add-date': orderby = 'mp.add_date' elif sorting == 'add-date-desc': orderby = 'mp.add_date DESC' elif sorting == 'release-date': orderby = 'md.release_date' elif sorting == 'release-date-desc': orderby = 'md.release_date DESC' else: orderby = 'md.title' # TODO: order by genres, unwatched folder... conn = db.get_db() cur = conn.cursor() view_results = [] query = f'''SELECT md.*, pc.path FROM query_details md JOIN path_cache pc ON pc.movie_id=md.movie_id ORDER BY {orderby}''' # TODO: should probably loop on movie_id, and add a list of paths because # there can be multiple paths associated with a movie_id for row in cur.execute(query): vr = buildViewResult(row) vr.rel_path = row['path'] view_results.append(vr) return render_template('tmdb/list_movie_test.html', view_results = view_results, url_prefix = current_app.config['VIDEO_URL_PREFIX'] ) @bp.route('/manual_query', methods=['GET', 'POST']) def manual_query(): query = query_data() if 'query' in request.form: query.raw_query_str = request.form['query'] q_res = doQueryAlgorithm(query) errors = q_res['errors'] view_results = q_res['results'] if errors: view_results.clear() flash(errors) for e in errors: current_app.logger.error(e) return render_template( 'tmdb/query.html', view_data = view_results, debug_str = f'{util.dumpObject(query)} \n {util.dumpObject(view_results)} \n' ) @bp.route('/query_media_json') def query_media_json(): query = query_data() results = [] if 'query' in request.args and 'rel_path' in request.args: query.raw_query_str = request.args['query'] query.rel_path = request.args['rel_path'] q_res = doQueryAlgorithm(query) errors = q_res['errors'] results = q_res['results'] current_app.logger.debug(f'results length: {len(results)}') else: errors = ['required parameters: query, rel_path'] if errors: for e in errors: current_app.logger.error(e) return jsonify(errors=errors) res = Response(json.dumps(results, cls=result_encoder)) res.headers.set('Content-Type', 'application/json') return res @bp.route('/query_path_json') def query_path_json(): errors = [] if not 'rel_path' in request.args: errors.append(f'required parameters: rel_path') else: path = request.args['rel_path'] cur = db.get_db().cursor() # NOTE: need to make an extra query to see what our id column is, tv_id or movie_id mr = cur.execute('SELECT media_type FROM path_cache WHERE path=?', (path, )).fetchone() if mr != None: id_column = getIDColumn(mr['media_type']) q = f'''SELECT qd.*, pc.* FROM path_cache pc JOIN query_details qd ON qd.{id_column} = pc.{id_column} WHERE pc.path = ? LIMIT 1''' cur.execute(q, (path, )) row = cur.fetchone() if errors: for e in errors: current_app.logger.error(e) res = Response(json.dumps(buildViewResult(row), cls=result_encoder)) res.headers.set('Content-Type', 'application/json') return res @bp.route('/create_path_mapping', methods=['POST']) def create_path_mapping(): message = '' errors = [] if not ('rel_path' in request.form and 'media_type' in request.form and 'id' in request.form ): errors.append(f'required parameters: rel_path, media_type, id') else: id = request.form['id'] media_type = request.form['media_type'] rel_path = request.form['rel_path'] id_column = getIDColumn(media_type) conn = db.get_db() cur = conn.cursor() current_app.logger.debug(f'mapping path "{rel_path}" for id: {id}') try: cur.execute(f'INSERT INTO path_cache({id_column}, media_type, path, add_date) ' f'VALUES(?,?,?, datetime())', (id, media_type, rel_path )) except DatabaseError as e: errors.append(str(e)) if len(errors) == 0: conn.commit() message = f'Successfully added mapping for path "{rel_path}"' if errors: for e in errors: current_app.logger.error(e) return jsonify({'errors':errors, 'message':message}) ################### # helpers def doQueryAlgorithm(query:query_data) -> dict: errors, view_results = [], [] response = response_data() # TODO: if query_obj.media_type is still 'UNKNOWN' from parseQueryStr(), make queries to # both tmdb endpoints (movie and tv) if query.raw_query_str and parseQueryStr(query): if isQueryCached(query.parsed_query_str): view_results = makeLocalQuery(query) current_app.logger.debug(f'view_results length: {len(view_results)}') else: makeTMDBQuery(query, response) try: response.parsed_response = json.loads(response.response_body) except json.JSONDecodeError as e: errors.append(e) if 'total_results' in response.parsed_response: response.result_count = response.parsed_response['total_results'] # TODO: it's possible for the query results to span multiple 'pages' # see) https://developers.themoviedb.org/3/search/search-movies if 'results' in response.parsed_response: for result in response.parsed_response['results']: view_results.append(parseTMDBResult(result, query.media_type)) storeTMDBQueryResults(query, view_results) errors.extend(query.errors) errors.extend(response.errors) return {'errors': errors, 'results': view_results} # TODO: need a way to correctly parse titles like: "2001: A Space Odyssey (1968)" # with legitimate dates in the title def parseQueryStr(query_obj): s = query_obj.raw_query_str.replace('.', ' ').strip() s = re.sub('[(){}\[\]]', '', s).strip() # NOTE: remove some common characters current_year = date.today().year # NOTE: find all 4 digit numbers to match the year of queries with 4 digit numbers in the title # ex) "2036.Origin.Unknown.2018" regex = re.compile('[0-9]{4}') for m in regex.finditer(s): i = int(m[0]) if i > 1878 and i <= current_year: query_obj.parsed_query_str = s[0:m.start()].strip() # NOTE: slice query string to year match query_obj.movie_year = i query_obj.media_type = media_type.MOVIE return True # NOTE: if not a movie, check for a string that looks like a tv show if query_obj.media_type == media_type.UNKNOWN: m = re.compile(r's[0-9]{1,2}|season ?[0-9]{1,2}', re.IGNORECASE).search(s) if m: query_obj.parsed_query_str = s[0:m.start()].strip() query_obj.media_type = media_type.TV_SHOW return True current_app.logger.warning(f'unable to guess media type for "{query_obj.raw_query_str}"') return False def isQueryCached(query_str): conn = db.get_db() cur = conn.cursor() cur.execute('SELECT COUNT(*) FROM query_cache WHERE query=?', (query_str,)) count = cur.fetchone()[0] current_app.logger.debug(f'query_cache count: {count}, query: "{query_str}"') return (count > 0) def getIDColumn(mediaType:int) -> str: if int(mediaType) == media_type.MOVIE: return 'movie_id' elif int(mediaType) == media_type.TV_SHOW: return 'tv_id' return 'unknown' # NOTE: returns list of 'view_result' objects that match the query_obj.parsed_query_str # stored in the local db cache def makeLocalQuery(query_obj): conn = db.get_db() cur = conn.cursor() results = [] id_column = getIDColumn(query_obj.media_type) query = f'''SELECT qd.* FROM query_cache mq JOIN query_mapping map ON mq.query_id = map.query_id JOIN query_details qd ON qd.{id_column} = map.{id_column} WHERE mq.query = ?''' for row in cur.execute(query, (query_obj.parsed_query_str, )): results.append(buildViewResult(row)) return results def buildViewResult(db_row): vr = view_result() vr.media_type = db_row['media_type'] vr.id = db_row[getIDColumn(vr.media_type)] vr.title = db_row['title'] vr.overview = db_row['overview'] vr.release_date = db_row['release_date'] vr.original_language = db_row['original_language'] if vr.media_type == media_type.MOVIE: vr.tmdb_link = f'{TMDB_MOVIE_LINK_URL}{vr.id}' elif vr.media_type == media_type.TV_SHOW: vr.tmdb_link = f'{TMDB_TV_LINK_URL}{vr.id}' if db_row['poster_path'] != None: vr.poster_path = db_row['poster_path'] return vr # TODO: need to check response headers for rate limiting requests def makeTMDBQuery(query_obj, response_obj): if query_obj.media_type == media_type.MOVIE: search_part = TMDB_SEARCH_MOVIE elif query_obj.media_type == media_type.TV_SHOW: search_part = TMDB_SEARCH_TV else: query_obj.errors.append('query type is not set') return False current_app.logger.info( f'making remote query to TMDB API: ' f'{query_obj.media_type.name}, "{query_obj.parsed_query_str}"' ) request_headers = {"Accept": "text/html,application/xhtml+xml,application/xml"} query_obj.query_params = urllib.parse.urlencode( { 'api_key': current_app.config['TMDB_API_KEY'], 'query': query_obj.parsed_query_str, 'year': query_obj.movie_year, } ) query_obj.full_url = TMDB_API_URL + search_part + '?' + query_obj.query_params context = ssl.create_default_context() req = urllib.request.Request(query_obj.full_url, None, request_headers) ct_header = '' try: response = urllib.request.urlopen(req, context=context) except HTTPError as e: query_obj.errors.append(e) return False except URLError as e: query_obj.errors.append(e) return False for k, v in response.info().items(): response_obj.response_headers.append((k, v)) if k == 'Content-Type': ct_header = str(k + ':' + v) response_obj.response_status = response.status if 'charset' in ct_header: charset = cgi.parse_header(ct_header)[1]['charset'] response_obj.response_body = response.read().decode(charset) else: response_obj.errors.append('no charset key in ct_header') response_obj.response_body = response.read() return False return True # NOTE: @param result is parsed JSON object from tmdb API def parseTMDBResult(result:object, media_type:int) -> view_result: parsed = view_result() try: parsed.id = result['id'] parsed.overview = result['overview'] parsed.genre_ids = result['genre_ids'] parsed.original_language = result['original_language'] parsed.poster_path = result['poster_path'] parsed.media_type = media_type if media_type == media_type.MOVIE: parsed.tmdb_link = TMDB_MOVIE_LINK_URL + str(result['id']) parsed.title = result['title'] parsed.release_date = result['release_date'] elif media_type == media_type.TV_SHOW: parsed.tmdb_link = TMDB_TV_LINK_URL + str(result['id']) parsed.title = result['name'] parsed.release_date = result['first_air_date'] except KeyError as e: current_app.logger.warning(f'missing key in JSON for "{parsed.title}": {e}') return parsed def storeTMDBQueryResults(query_obj, result_list): conn = db.get_db() cur = conn.cursor() query_id = 0 current_app.logger.debug(f'adding query string ' f'"{query_obj.parsed_query_str}" into local cache') try: cur.execute('BEGIN TRANSACTION') cur.execute('INSERT INTO query_cache(query, year) VALUES (?, ?)', (query_obj.parsed_query_str, query_obj.movie_year) ) query_id = cur.lastrowid except DatabaseError as e: query_obj.errors.append(e) for r in result_list: id_column = getIDColumn(r.media_type) # NOTE: check stored details before trying to add to local db to avoid unique key # constraint error cur.execute(f'SELECT COUNT(*) FROM query_details WHERE {id_column}=?', (r.id, )) count = cur.fetchone()[0] if count > 0: current_app.logger.debug( f'skipping result that is already saved. id: {r.id}, title: {r.title}' ) break try: current_app.logger.debug(f'adding query details for "{r.title}"') cur.execute(f'INSERT INTO query_details({id_column}, ' + 'media_type, title, overview, release_date, original_language, poster_path)' + ' VALUES (?,?,?,?,?,?,?)', (r.id, r.media_type, r.title, r.overview, r.release_date, r.original_language, r.poster_path) ) cur.execute(f'INSERT INTO query_mapping(query_id, {id_column}, media_type)' + ' VALUES (?,?,?)', (query_id, r.id, r.media_type) ) except DatabaseError as e: query_obj.errors.append(f'database error \'{e}\'') if len(query_obj.errors) == 0: conn.commit() else: conn.rollback()