From feceb91f4ecc1fac855fb3ed92bae396c21b09ff Mon Sep 17 00:00:00 2001 From: cinnaboot Date: Tue, 22 Oct 2019 00:36:38 -0400 Subject: [PATCH] add support for tmdb tv API endpoint --- video_metadata/schema.sql | 6 +- video_metadata/static/file_listing.js | 11 ++- video_metadata/util.py | 5 + video_metadata/views/tmdb.py | 129 +++++++++++++++----------- 4 files changed, 92 insertions(+), 59 deletions(-) diff --git a/video_metadata/schema.sql b/video_metadata/schema.sql index e1ad6e4..7ea2d5c 100644 --- a/video_metadata/schema.sql +++ b/video_metadata/schema.sql @@ -8,9 +8,9 @@ CREATE TABLE query_cache( DROP TABLE IF EXISTS query_details; CREATE TABLE query_details( - media_type text, -- NOTE: tv, movie, or unkown movie_id integer, -- NOTE: only one of movie_id, tv_id should be set tv_id integer, + media_type integer, -- NOTE: 0=UNKNOW, 1=MOVIE, 2=TV_SHOW title text, overview text, release_date date, @@ -21,8 +21,9 @@ CREATE TABLE query_details( DROP TABLE IF EXISTS query_mapping; CREATE TABLE query_mapping( query_id integer, - movie_id integer, -- NOTE: only one of movie_id, tv_id should be set + movie_id integer, tv_id integer, + media_type integer, foreign key(query_id) references query_cache(query_id), foreign key(movie_id) references query_details(movie_id), foreign key(tv_id) references query_details(tv_id) @@ -33,6 +34,7 @@ CREATE TABLE path_cache( path text unique not null, movie_id integer, tv_id integer, + media_type integer, add_date datetime, -- NOTE: date the path was mapped foreign key(movie_id) references query_details(movie_id), foreign key(tv_id) references query_details(tv_id) diff --git a/video_metadata/static/file_listing.js b/video_metadata/static/file_listing.js index e60c37f..4479632 100644 --- a/video_metadata/static/file_listing.js +++ b/video_metadata/static/file_listing.js @@ -1,8 +1,8 @@ // TODO: could get path constants from DOM in jinja template const FOLDER_CONTENTS_PATH = '/pvdb/file_listing/folder_contents_json/'; -const QUERY_PATH = '/pvdb/tmdb/query_movie_json'; -const CREATE_MAPPING_PATH = '/pvdb/tmdb/create_mapping'; +const QUERY_PATH = '/pvdb/tmdb/query_media_json'; +const CREATE_MAPPING_PATH = '/pvdb/tmdb/create_path_mapping'; const PATH_PATH = '/pvdb/tmdb/query_path_json'; const VIDEO_PREFIX = '/video/'; @@ -99,7 +99,7 @@ function openQueryDialog(dataset, element) { for (let result of json) { blurb = createBlurb(result); blurb.querySelector('.mapping_link').addEventListener('click', function(e) { - submitFileMapping(e, result.id, dataset.rel_path, element); + submitFileMapping(e, result.id, result.media_type, dataset.rel_path, element); }); tclone.querySelector('#query_result_container').appendChild(blurb); } @@ -126,13 +126,14 @@ function createBlurb(details) { return qtclone; } -async function submitFileMapping(click_event, movie_id, rel_path, element) { +async function submitFileMapping(click_event, id, media_type, rel_path, element) { click_event.preventDefault(); // NOTE: remove the query box document.querySelector('body').removeChild(document.querySelector('#query_result_container')); query_str = CREATE_MAPPING_PATH; params = new FormData(); - params.append('movie_id', movie_id); + params.append('id', id); + params.append('media_type', media_type); params.append('rel_path', rel_path); let json = null; diff --git a/video_metadata/util.py b/video_metadata/util.py index 6e4ca42..680ec21 100644 --- a/video_metadata/util.py +++ b/video_metadata/util.py @@ -29,3 +29,8 @@ def getFunctionName(): # TODO: make output prettier def dumpObject(obj): return yaml.dump(obj) + + +def getLineNum() -> int: + from inspect import currentframe, getframeinfo + return currentframe().f_back.f_lineno diff --git a/video_metadata/views/tmdb.py b/video_metadata/views/tmdb.py index 40703c4..801db2c 100644 --- a/video_metadata/views/tmdb.py +++ b/video_metadata/views/tmdb.py @@ -2,7 +2,7 @@ import cgi from dataclasses import dataclass, field from datetime import date, datetime -from enum import Enum +from enum import IntEnum import json import re from sqlite3 import DatabaseError @@ -37,7 +37,7 @@ bp = Blueprint('tmdb', __name__, url_prefix='/tmdb') #################### # data structures -class media_type(Enum): +class media_type(IntEnum): UNKNOWN = 0 MOVIE = 1 TV_SHOW = 2 @@ -64,6 +64,7 @@ class response_data: @dataclass class view_result: + media_type:str = 'unknown' id: int = 0 title: str = '' release_date: str = '' @@ -162,8 +163,8 @@ def manual_query(): ) -@bp.route('/query_movie_json') -def query_movie_json(): +@bp.route('/query_media_json') +def query_media_json(): query = query_data() results = [] @@ -191,20 +192,28 @@ def query_movie_json(): def query_path_json(): errors = [] - if 'rel_path' in request.args: + if not 'rel_path' in request.args: + errors.append(f'{util.getFunctionName()}, required parameters: rel_path') + else: path = request.args['rel_path'] cur = db.get_db().cursor() - q = '''SELECT - md.*, pc.* - FROM - path_cache pc - JOIN query_details md ON md.movie_id = pc.movie_id - WHERE - pc.path = ? - LIMIT 1''' - - cur.execute(q, (path, )) - row = cur.fetchone() + + # NOTE: need to make an extra query to see what our id column is, tv_id or movie_id + mr = cur.execute('SELECT media_type FROM path_cache WHERE path=?', (path, )).fetchone() + if mr != None: + id_column = getIDColumn(mr['media_type']) + + q = f'''SELECT + qd.*, pc.* + FROM + path_cache pc + JOIN query_details qd ON qd.{id_column} = pc.{id_column} + WHERE + pc.path = ? + LIMIT 1''' + + cur.execute(q, (path, )) + row = cur.fetchone() if errors: for e in errors: @@ -215,23 +224,28 @@ def query_path_json(): return res -# TODO: add option for tv shows -@bp.route('/create_mapping', methods=['POST']) -def create_mapping(): - movie_id = request.form['movie_id'] - rel_path = request.form['rel_path'] +@bp.route('/create_path_mapping', methods=['POST']) +def create_path_mapping(): message = '' errors = [] - if not movie_id or not rel_path: - errors.append('invalid parameter') + if not ('rel_path' in request.form + and 'media_type' in request.form + and 'id' in request.form + ): + errors.append(f'{util.getFunctionName()}, required parameters: rel_path, media_type, id') else: + id = request.form['id'] + media_type = request.form['media_type'] + rel_path = request.form['rel_path'] + id_column = getIDColumn(media_type) conn = db.get_db() cur = conn.cursor() - current_app.logger.debug(f'{util.getFunctionName()}, mapping path "{rel_path}" for id: {movie_id}') + current_app.logger.debug(f'{util.getFunctionName()}, mapping path "{rel_path}" for id: {id}') try: - cur.execute('INSERT INTO path_cache(movie_id, path, add_date) VALUES(?, ?, datetime())', - (movie_id, rel_path )) + cur.execute(f'INSERT INTO path_cache({id_column}, media_type, path, add_date) ' + f'VALUES(?,?,?, datetime())', + (id, media_type, rel_path )) except DatabaseError as e: errors.append(str(e)) @@ -330,23 +344,33 @@ def isQueryCached(query_str): return (count > 0) +def getIDColumn(mediaType:int) -> str: + if int(mediaType) == media_type.MOVIE: + return 'movie_id' + elif int(mediaType) == media_type.TV_SHOW: + return 'tv_id' + + return 'unknown' + + # NOTE: returns list of 'view_result' objects that match the query_obj.parsed_query_str # stored in the local db cache def makeLocalQuery(query_obj): conn = db.get_db() cur = conn.cursor() results = [] + id_column = getIDColumn(query_obj.media_type) - fancy_query = '''SELECT - md.* - FROM - query_cache mq - JOIN query_mapping map ON mq.query_id = map.query_id - JOIN query_details md ON md.movie_id = map.movie_id - WHERE - mq.query = ?''' + query = f'''SELECT + qd.* + FROM + query_cache mq + JOIN query_mapping map ON mq.query_id = map.query_id + JOIN query_details qd ON qd.{id_column} = map.{id_column} + WHERE + mq.query = ?''' - for row in cur.execute(fancy_query, (query_obj.parsed_query_str, )): + for row in cur.execute(query, (query_obj.parsed_query_str, )): results.append(buildViewResult(row)) return results @@ -354,7 +378,8 @@ def makeLocalQuery(query_obj): def buildViewResult(db_row): vr = view_result() - vr.id = db_row['movie_id'] + vr.media_type = db_row['media_type'] + vr.id = db_row[getIDColumn(vr.media_type)] vr.title = db_row['title'] vr.overview = db_row['overview'] vr.release_date = db_row['release_date'] @@ -367,7 +392,6 @@ def buildViewResult(db_row): return vr -# TODO: add info and debug logging for network requests # TODO: need to check response headers for rate limiting requests def makeTMDBQuery(query_obj, response_obj): if query_obj.media_type == media_type.MOVIE: @@ -380,7 +404,7 @@ def makeTMDBQuery(query_obj, response_obj): current_app.logger.info( f'{util.getFunctionName()}, making remote query to TMDB API: ' - f'{query_obj.media_type}, "{query_obj.parsed_query_str}"' + f'{query_obj.media_type.name}, "{query_obj.parsed_query_str}"' ) request_headers = {"Accept": "text/html,application/xhtml+xml,application/xml"} query_obj.query_params = urllib.parse.urlencode( @@ -424,7 +448,7 @@ def makeTMDBQuery(query_obj, response_obj): # NOTE: @param result is parsed JSON object from tmdb API -def parseTMDBResult(result, media_type): +def parseTMDBResult(result:object, media_type:int) -> view_result: parsed = view_result() try: @@ -433,6 +457,7 @@ def parseTMDBResult(result, media_type): parsed.genre_ids = result['genre_ids'] parsed.original_language = result['original_language'] parsed.poster_path = result['poster_path'] + parsed.media_type = media_type if media_type == media_type.MOVIE: parsed.tmdb_link = TMDB_MOVIE_LINK_URL + str(result['id']) @@ -465,32 +490,32 @@ def storeTMDBQueryResults(query_obj, result_list): except DatabaseError as e: query_obj.errors.append(e) - for result in result_list: + for r in result_list: + id_column = getIDColumn(r.media_type) + # NOTE: check stored details before trying to add to local db to avoid unique key # constraint error - cur.execute('SELECT COUNT(*) FROM query_details WHERE movie_id=?', (result.id,)) + cur.execute(f'SELECT COUNT(*) FROM query_details WHERE {id_column}=?', (r.id, )) count = cur.fetchone()[0] if count > 0: current_app.logger.debug(f'{util.getFunctionName()}, ' - f'skipping result that is already saved. id: {result.id}, title: {result.title}' + f'skipping result that is already saved. id: {r.id}, title: {r.title}' ) break try: - current_app.logger.debug(f'{util.getFunctionName()}, adding query details for "{result.title}"') - # TODO: need to add tv_id to query_cache - cur.execute("INSERT INTO query_details(" - + "movie_id, title, overview, release_date, original_language, poster_path)" - + " VALUES (?,?,?,?,?,?)", - (result.id, result.title, result.overview, result.release_date, - result.original_language, result.poster_path) + current_app.logger.debug(f'{util.getFunctionName()}, adding query details for "{r.title}"') + cur.execute(f'INSERT INTO query_details({id_column}, ' + + 'media_type, title, overview, release_date, original_language, poster_path)' + + ' VALUES (?,?,?,?,?,?,?)', + (r.id, r.media_type, r.title, r.overview, r.release_date, r.original_language, r.poster_path) ) - cur.execute("INSERT INTO query_mapping(query_id, movie_id)" - + " VALUES (?,?)", (query_id, result.id) + cur.execute(f'INSERT INTO query_mapping(query_id, {id_column}, media_type)' + + ' VALUES (?,?,?)', (query_id, r.id, r.media_type) ) except DatabaseError as e: - query_obj.errors.append(f'{util.getFunctionName()}, database error \'{e}\'') + query_obj.errors.append(f'{util.getFunctionName()}():{util.getLineNum()}, database error \'{e}\'') if len(query_obj.errors) == 0: conn.commit()